DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.size() != 3 && arguments.size() != 4) throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + ", should be 3 or 4", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!isString(arguments[0])) throw Exception("First argument for function " + getName() + " (unit) must be String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!isDateOrDateTime(arguments[1])) throw Exception("Second argument for function " + getName() + " must be Date or DateTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!isDateOrDateTime(arguments[2])) throw Exception("Third argument for function " + getName() + " must be Date or DateTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 4 && !isString(arguments[3])) throw Exception("Fourth argument for function " + getName() + " (timezone) must be String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared<DataTypeInt64>(); }
SumMapArgs parseArguments(const std::string & name, const DataTypes & arguments) { if (arguments.size() < 2) throw Exception("Aggregate function " + name + " requires at least two arguments of Array type.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); const auto * array_type = checkAndGetDataType<DataTypeArray>(arguments[0].get()); if (!array_type) throw Exception("First argument for function " + name + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); DataTypePtr keys_type = array_type->getNestedType(); DataTypes values_types; values_types.reserve(arguments.size() - 1); for (size_t i = 1; i < arguments.size(); ++i) { array_type = checkAndGetDataType<DataTypeArray>(arguments[i].get()); if (!array_type) throw Exception("Argument #" + toString(i) + " for function " + name + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); values_types.push_back(array_type->getNestedType()); } return {std::move(keys_type), std::move(values_types)}; }
void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const { size_t key_size = storage.partition_key_sample.columns(); if (key_size == 0) { writeCString("tuple()", out); } else if (key_size == 1) { const DataTypePtr & type = storage.partition_key_sample.getByPosition(0).type; auto column = type->createColumn(); column->insert(value[0]); type->serializeAsText(*column, 0, out, format_settings); } else { DataTypes types; Columns columns; for (size_t i = 0; i < key_size; ++i) { const auto & type = storage.partition_key_sample.getByPosition(i).type; types.push_back(type); auto column = type->createColumn(); column->insert(value[i]); columns.push_back(std::move(column)); } DataTypeTuple tuple_type(types); auto tuple_column = ColumnTuple::create(columns); tuple_type.serializeText(*tuple_column, 0, out, format_settings); } }
DataTypes NamesAndTypesList::getTypes() const { DataTypes res; res.reserve(size()); for (const NameAndTypePair & column : *this) res.push_back(column.type); return res; }
DataTypePtr getReturnType(const DataTypes & arguments) const override { if (arguments.size() != 3) throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + ", should be 3.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (size_t i : ext::range(0, 3)) if (!typeid_cast<const DataTypeString *>(&*arguments[i])) throw Exception("Illegal type " + arguments[i]->getName() + " of argument of function " + getName() + ", must be String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared<DataTypeString>(); }
void DictionaryStructure::validateKeyTypes(const DataTypes & key_types) const { if (key_types.size() != key.value().size()) throw Exception{ "Key structure does not match, expected " + getKeyDescription(), ErrorCodes::TYPE_MISMATCH}; for (const auto i : ext::range(0, key_types.size())) { const auto & expected_type = (*key)[i].type->getName(); const auto & actual_type = key_types[i]->getName(); if (expected_type != actual_type) throw Exception{ "Key type at position " + std::to_string(i) + " does not match, expected " + expected_type + ", found " + actual_type, ErrorCodes::TYPE_MISMATCH}; } }
DataTypePtr FunctionCoalesce::getReturnTypeImpl(const DataTypes & arguments) const { DataTypes new_args; for (size_t i = 0; i < arguments.size(); ++i) { new_args.push_back(std::make_shared<DataTypeUInt8>()); new_args.push_back(arguments[i]); } new_args.push_back(std::make_shared<DataTypeNull>()); return FunctionMultiIf{}.getReturnTypeImpl(new_args); }
DataTypePtr FunctionCoalesce::getReturnTypeImpl(const DataTypes & arguments) const { /// Skip all NULL arguments. If any argument is non-Nullable, skip all next arguments. DataTypes filtered_args; filtered_args.reserve(arguments.size()); for (const auto & arg : arguments) { if (arg->onlyNull()) continue; filtered_args.push_back(arg); if (!arg->isNullable()) break; } DataTypes new_args; for (size_t i = 0; i < filtered_args.size(); ++i) { bool is_last = i + 1 == filtered_args.size(); if (is_last) { new_args.push_back(filtered_args[i]); } else { new_args.push_back(std::make_shared<DataTypeUInt8>()); new_args.push_back(removeNullable(filtered_args[i])); } } if (new_args.empty()) return std::make_shared<DataTypeNullable>(std::make_shared<DataTypeNothing>()); if (new_args.size() == 1) return new_args.front(); auto res = FunctionMultiIf{context}.getReturnTypeImpl(new_args); /// if last argument is not nullable, result should be also not nullable if (!new_args.back()->isNullable() && res->isNullable()) res = removeNullable(res); return res; }
void PelicanServerClientTest::test_getData() { Config config; config.setFromString( "<testconfig>" " <server host=\"127.0.0.1\"/>" "</testconfig>" ); Config::TreeAddress address; address << Config::NodeId("testconfig", ""); ConfigNode configNode = config.get(address); TestStreamAdapter streamAdapter; QString stream1("stream1"); QString version1("version1"); DataSpec req; req.addStreamData(stream1); { // Use Case: // data requirements are set, empty hash passed // Expect: throw QList<DataSpec> lreq; lreq.append(req); DataTypes dt; dt.setAdapter(stream1,&streamAdapter); dt.addData(lreq); PelicanServerClient client(configNode, dt, 0); QHash<QString, DataBlob*> dataHash; CPPUNIT_ASSERT_THROW(client.getData(dataHash), QString ); } { // Use Case: // Data Requirements do not match DataBlobs // Expect: // throw with a suitable message // setup the test QList<DataSpec> lreq; lreq.append(req); DataTypes dt; dt.setAdapter(stream1,&streamAdapter); dt.addData(lreq); PelicanServerClient client(configNode, dt, 0); QHash<QString, DataBlob*> dataHash; DataBlob db("DataBlob"); dataHash.insert("pelican/data2", &db); CPPUNIT_ASSERT_THROW(client.getData(dataHash), QString); } }
DataTypePtr getReturnTypeImpl(const DataTypes & args) const override { /// Arguments are the following: cond1, then1, cond2, then2, ... condN, thenN, else. auto for_conditions = [&args](auto && f) { size_t conditions_end = args.size() - 1; for (size_t i = 0; i < conditions_end; i += 2) f(args[i]); }; auto for_branches = [&args](auto && f) { size_t branches_end = args.size(); for (size_t i = 1; i < branches_end; i += 2) f(args[i]); f(args.back()); }; if (!(args.size() >= 3 && args.size() % 2 == 1)) throw Exception{"Invalid number of arguments for function " + getName(), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; /// Conditions must be UInt8, Nullable(UInt8) or Null. If one of conditions is Nullable, the result is also Nullable. bool have_nullable_condition = false; for_conditions([&](const DataTypePtr & arg) { const IDataType * nested_type; if (arg->isNullable()) { have_nullable_condition = true; if (arg->onlyNull()) return; const DataTypeNullable & nullable_type = static_cast<const DataTypeNullable &>(*arg); nested_type = nullable_type.getNestedType().get(); } else { nested_type = arg.get(); } if (!WhichDataType(nested_type).isUInt8()) throw Exception{"Illegal type " + arg->getName() + " of argument (condition) " "of function " + getName() + ". Must be UInt8.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; }); DataTypes types_of_branches; types_of_branches.reserve(args.size() / 2 + 1); for_branches([&](const DataTypePtr & arg) { types_of_branches.emplace_back(arg); }); DataTypePtr common_type_of_branches = getLeastSupertype(types_of_branches); return have_nullable_condition ? makeNullable(common_type_of_branches) : common_type_of_branches; }
void PelicanServerClientTestMT::test_getData() { try{ // set up the test server TestServiceAdapter serviceAdapter; TestStreamAdapter streamAdapter; TestServer server; QString stream1("stream1"); QString service1("service1"); QString version1("version1"); QString version2("version2"); DataRequirements reqStream1; reqStream1.addStreamData(stream1); QByteArray data1("pelican/data1"); QByteArray data2("pelican/data2"); unsigned int port = server.port(); // common client configuration Config config; config.setFromString( "", "<testconfig>" " <server host=\"127.0.0.1\"/>" "</testconfig>" ); Config::TreeAddress address; address << Config::NodeId("server", ""); address << Config::NodeId("testconfig", ""); ConfigNode configNode = config.get(address); { // Use Case: // request for Service Data only // Expect : // throw DataChunk sd(service1, version1, data1 ); CPPUNIT_ASSERT_EQUAL( (long)data1.size(), (long)sd.size() ); // setup the test DataRequirements req; req.addServiceData(service1); QList<DataRequirements> lreq; lreq.append(req); DataTypes dt; dt.addData(lreq); dt.setAdapter(service1, &serviceAdapter); PelicanServerClient client(configNode, dt, 0); client.setPort(port); QHash<QString, DataBlob*> dataHash; DataBlob db("DataBlob"); dataHash.insert(service1, &db); CPPUNIT_ASSERT_THROW( client.getData(dataHash), QString ); } { // Use Case: // Single Request for an existing stream dataset // no service data // Expect: // return the required data stream StreamData sd(stream1, version1, data1 ); CPPUNIT_ASSERT_EQUAL( (long)data1.size(), (long)sd.size() ); server.serveStreamData(sd); // setup the test QList<DataRequirements> lreq; lreq.append(reqStream1); DataTypes dt; dt.addData(lreq); dt.setAdapter(stream1, &streamAdapter); PelicanServerClient client(configNode, dt, 0); client.setPort(port); QHash<QString, DataBlob*> dataHash; TestDataBlob db; dataHash.insert(stream1, &db); client.getData(dataHash); CPPUNIT_ASSERT_EQUAL(version1.toStdString(), db.version().toStdString() ); CPPUNIT_ASSERT_EQUAL( std::string(data1.data()), std::string(db.data()) ); } { // Use Case: // Single Request for an existing stream dataset // with existing service data // Expect: // return the required data stream DataChunk servd(service1, version2, data2 ); server.serveServiceData(servd); StreamData sd(stream1, version1, data1 ); CPPUNIT_ASSERT_EQUAL( (long)data1.size(), (long)sd.size() ); server.serveStreamData(sd); // setup the test DataRequirements req; req.addServiceData(service1); req.addStreamData(stream1); QList<DataRequirements> lreq; lreq.append(req); DataTypes dt; dt.addData(lreq); dt.setAdapter(stream1, &streamAdapter); dt.setAdapter(service1, &serviceAdapter); PelicanServerClient client(configNode, dt, 0); client.setPort(port); QHash<QString, DataBlob*> dataHash; TestDataBlob db; TestDataBlob db_service; dataHash.insert(stream1, &db); dataHash.insert(service1, &db_service); client.getData(dataHash); CPPUNIT_ASSERT_EQUAL( version1.toStdString(), db.version().toStdString() ); CPPUNIT_ASSERT_EQUAL( version2.toStdString(), db_service.version().toStdString() ); CPPUNIT_ASSERT_EQUAL( std::string(data2.data()), std::string(db_service.data()) ); CPPUNIT_ASSERT_EQUAL( std::string(data1.data()), std::string(db.data()) ); } } catch(const QString& e) { std::cerr << "error: uncaught exception" << e.toStdString(); throw e; } }
AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const DataTypes & argument_types, int recursion_level) const { auto it = aggregate_functions.find(name); if (it != aggregate_functions.end()) { const auto & desc = it->second; const auto & creator = desc.creator; return creator(name, argument_types); } else if ((recursion_level == 0) && endsWith<SuffixState>(name)) { /// Для агрегатных функций вида aggState, где agg - имя другой агрегатной функции. AggregateFunctionPtr nested = get(trimRight<SuffixState>(name), argument_types, recursion_level + 1); return createAggregateFunctionState(nested); } else if ((recursion_level <= 1) && endsWith<SuffixMerge>(name)) { /// Для агрегатных функций вида aggMerge, где agg - имя другой агрегатной функции. if (argument_types.size() != 1) throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); const DataTypeAggregateFunction * function = typeid_cast<const DataTypeAggregateFunction *>(&*argument_types[0]); if (!function) throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); AggregateFunctionPtr nested = get(trimRight<SuffixMerge>(name), function->getArgumentsDataTypes(), recursion_level + 1); if (nested->getName() != function->getFunctionName()) throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return createAggregateFunctionMerge(nested); } else if ((recursion_level <= 2) && endsWith<SuffixIf>(name)) { if (argument_types.empty()) throw Exception{ "Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH }; /// Для агрегатных функций вида aggIf, где agg - имя другой агрегатной функции. DataTypes nested_dt = argument_types; nested_dt.pop_back(); AggregateFunctionPtr nested = get(trimRight<SuffixIf>(name), nested_dt, recursion_level + 1); return createAggregateFunctionIf(nested); } else if ((recursion_level <= 3) && endsWith<SuffixArray>(name)) { /// Для агрегатных функций вида aggArray, где agg - имя другой агрегатной функции. size_t num_agruments = argument_types.size(); DataTypes nested_arguments; for (size_t i = 0; i < num_agruments; ++i) { if (const DataTypeArray * array = typeid_cast<const DataTypeArray *>(&*argument_types[i])) nested_arguments.push_back(array->getNestedType()); else throw Exception("Illegal type " + argument_types[i]->getName() + " of argument #" + toString(i + 1) + " for aggregate function " + name + ". Must be array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } /// + 3, чтобы ни один другой модификатор не мог идти перед Array AggregateFunctionPtr nested = get(trimRight<SuffixArray>(name), nested_arguments, recursion_level + 3); return createAggregateFunctionArray(nested); } else throw Exception("Unknown aggregate function " + name, ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION); }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { return arguments.at(0); }
void PelicanServerClientTest::test_response() { try { TestServiceAdapter serviceAdapter; TestStreamAdapter streamAdapter; ConfigNode configNode; configNode.setFromString( "<testconfig>" " <server host=\"127.0.0.1\"/>" "</testconfig>" ); QString stream1("stream1"); QString version1("v1"); QString service1("service1"); QString serviceVersion1("sv1"); DataSpec reqStream1; reqStream1.addStreamData(stream1); QByteArray data1("data1"); { // Use Case // receive a StreamData response (single stream, no service data) boost::shared_ptr<ServerResponse> res( new StreamDataResponse ); static_cast<StreamDataResponse*>(res.get())->setStreamData( new StreamData(stream1,version1, data1.size()) ); // set up the adapters and client QList<DataSpec> lreq; lreq.append(reqStream1); DataTypes dt; dt.setAdapter(stream1,&streamAdapter); dt.addData(lreq); PelicanServerClient client(configNode, dt, 0); // set up the memory for receiving data TestDataBlob db; QHash<QString, DataBlob*> dataHash; dataHash.insert(stream1, &db); SocketTester st; QTcpSocket& sock = st.send(data1); QHash<QString, DataBlob*> vhash = client._response( sock , res, dataHash ); CPPUNIT_ASSERT( vhash == dataHash ); CPPUNIT_ASSERT_EQUAL( version1.toStdString(), vhash[stream1]->version().toStdString() ); CPPUNIT_ASSERT_EQUAL( std::string( data1.data() ) , std::string( static_cast<TestDataBlob*>(vhash[stream1])->data() ) ); } QByteArray data2("data2"); { // Use Case // ServiceData response - single service data boost::shared_ptr<ServerResponse> res( new ServiceDataResponse ); static_cast<ServiceDataResponse*>(res.get())->addData( new DataChunk(service1,serviceVersion1, data2.size()) ); // set up the memory for receiving data TestDataBlob db; QHash<QString, DataBlob*> dataHash; dataHash.insert(service1, &db); // set up the adapters and client DataSpec req; req.addStreamData(service1); QList<DataSpec> lreq; lreq.append(req); DataTypes dt; dt.setAdapter(service1,&serviceAdapter); dt.addData(lreq); PelicanServerClient client(configNode, dt, 0); SocketTester st; QTcpSocket& sock = st.send(data2); QHash<QString, DataBlob*> vhash = client._response( sock , res, dataHash ); CPPUNIT_ASSERT( vhash == dataHash ); CPPUNIT_ASSERT_EQUAL( serviceVersion1.toStdString(), vhash[service1]->version().toStdString() ); CPPUNIT_ASSERT_EQUAL( std::string( data2.data() ) , std::string( static_cast<TestDataBlob*>(vhash[service1])->data() ) ); } QString service2("service2"); { // Use Case // ServiceData response - multiple service data boost::shared_ptr<ServerResponse> res( new ServiceDataResponse ); static_cast<ServiceDataResponse*>(res.get())->addData( new DataChunk(service1,version1, data1.size()) ); static_cast<ServiceDataResponse*>(res.get())->addData( new DataChunk(service2,serviceVersion1, data2.size()) ); // set up the memory for receiving data TestDataBlob db; TestDataBlob db2; QHash<QString, DataBlob*> dataHash; dataHash.insert(service1, &db); dataHash.insert(service2, &db2); // set up the adapters and client DataSpec req; req.addStreamData(service1); QList<DataSpec> lreq; lreq.append(req); DataTypes dt; dt.setAdapter(service1,&serviceAdapter); dt.setAdapter(service2,&serviceAdapter); dt.addData(lreq); PelicanServerClient client(configNode, dt, 0); SocketTester st; QTcpSocket& sock = st.send( data1 + data2 ); QHash<QString, DataBlob*> vhash = client._response( sock , res, dataHash ); CPPUNIT_ASSERT( vhash == dataHash ); CPPUNIT_ASSERT_EQUAL( version1.toStdString(), vhash[service1]->version().toStdString() ); CPPUNIT_ASSERT_EQUAL( serviceVersion1.toStdString(), vhash[service2]->version().toStdString() ); CPPUNIT_ASSERT_EQUAL( std::string( data1.data() ) , std::string( static_cast<TestDataBlob*>(vhash[service1])->data() ) ); CPPUNIT_ASSERT_EQUAL( std::string( data2.data() ) , std::string( static_cast<TestDataBlob*>(vhash[service2])->data() ) ); } // { // // Use Case // // receive a StreamData response with associated service data (single stream, single service data) // // no existing service data exists // boost::shared_ptr<ServerResponse> res( new StreamDataResponse ); // StreamData* sd = new StreamData(stream1,version1, data1.size()); // sd->addAssociatedData(boost::shared_ptr<Data>(new Data(service1,serviceVersion1, data2.size()) ) ); // static_cast<StreamDataResponse*>(res.get())->setStreamData( sd ); // // // set up the adapters and client // DataSpec req; // req.addStreamData(stream1); // req.addServiceData(service1); // QList<DataSpec> lreq; // lreq.append(req); // DataTypes dt; // dt.setAdapter(stream1,&streamAdapter); // dt.setAdapter(service1,&serviceAdapter); // dt.addData(lreq); // PelicanServerClient client(configNode, dt, 0); // // // set up the memory for receiving data // TestDataBlob db; // TestDataBlob db2; // QHash<QString, DataBlob*> dataHash; // dataHash.insert(stream1, &db); // dataHash.insert(service1, &db2); // // SocketTester st; // QTcpSocket& sock = st.send(data1); // try { // QHash<QString, DataBlob*> vhash = client._response( sock , res, dataHash ); // } catch (const QString& e) { // CPPUNIT_FAIL(e.toStdString()); // } // CPPUNIT_ASSERT( vhash == dataHash ); // CPPUNIT_ASSERT_EQUAL( version1.toStdString(), vhash[stream1]->version().toStdString() ); // CPPUNIT_ASSERT_EQUAL( serviceVersion1.toStdString(), vhash[service1]->version().toStdString() ); // CPPUNIT_ASSERT_EQUAL( std::string( data1.data() ) , std::string( static_cast<TestDataBlob*>( vhash[stream1])->data() ) ); // CPPUNIT_ASSERT_EQUAL( std::string( data2.data() ) , std::string( static_cast<TestDataBlob*>( vhash[service1])->data() ) ); // } } catch ( const QString& e ) { CPPUNIT_FAIL( e.toStdString() ); } }
void FileDataClientTest::test_method() { try { Config config; config.setFromString( "<testconfig>" " <file name=\"testfile.xml\"/>" "</testconfig>" ); Config::TreeAddress address; address << Config::NodeId("testconfig", ""); ConfigNode configNode = config.get(address); /* No longer relevant { // Use Case: // Empty data requirements // Expect: throw with a suitable complaint // setup the test QList<DataSpec> lreq; DataTypes dt; dt.addData(lreq); FileDataClient* client = 0; CPPUNIT_ASSERT_THROW(client = new FileDataClient(configNode, dt, 0), QString); delete client; } */ TestStreamAdapter streamAdapter; QString stream1("stream1"); QString version1("version1"); DataSpec req; req.addStreamData(stream1); { // Use Case: // data requirements are set, empty hash passed // Expect: throw QList<DataSpec> lreq; lreq.append(req); DataTypes dt; dt.setAdapter(stream1,&streamAdapter); dt.addData(lreq); FileDataClient client(configNode, dt, 0); QHash<QString, DataBlob*> dataHash; CPPUNIT_ASSERT_THROW(client.getData(dataHash), QString ); } { // Use Case: // Specifiy a single file and associate it with a Stream // correct hash is passed // Expect: // Data in file to be streamed DataTypes types; types.setAdapter(stream1,&streamAdapter); types.addData(req); FileDataClient client(configNode, types, &config); QHash<QString, DataBlob*> dataHash; DataBlob db("DataBlob"); dataHash.insert(stream1, &db); client.getData(dataHash); } } catch( QString& e ) { CPPUNIT_FAIL(e.toStdString()); } }