uint64_t S3BucketReader::readWithoutHeaderLine(char* buf, uint64_t count) { char* current = NULL; char* end = NULL; char* currentEOL = eolString; // check one char at a time while (*currentEOL != '\0') { if (current == end) { uint64_t readCount = this->upstreamReader->read(buf, count); // we have reach the end of file but found no matching EOL. if (readCount == 0) { S3WARN("%s", "Reach end of file before matching line terminator"); return 0; } current = buf; end = buf + readCount; } // skip until we met next newline char for (; current != end; current++) { if (*current == *currentEOL) { currentEOL++; current++; break; } else { currentEOL = eolString; } } } // move remained data to front. uint64_t remain = end - current; char* p = buf; while (current != end) { *p = *current; p++; current++; } return remain; }
S3Params InitConfig(const string& urlWithOptions) { #ifdef S3_STANDALONE s3ext_segid = 0; s3ext_segnum = 1; #else s3ext_segid = GpIdentity.segindex; s3ext_segnum = GpIdentity.numsegments; #endif if (s3ext_segid == -1 && s3ext_segnum > 0) { s3ext_segid = 0; s3ext_segnum = 1; } string sourceUrl = TruncateOptions(urlWithOptions); S3_CHECK_OR_DIE(!sourceUrl.empty(), S3RuntimeError, "URL not found from location string"); string configPath = GetOptS3(urlWithOptions, "config"); if (configPath.empty()) { S3WARN("The 'config' parameter is not provided, use default value 's3/s3.conf'."); configPath = "s3/s3.conf"; } string configSection = GetOptS3(urlWithOptions, "section"); if (configSection.empty()) { configSection = "default"; } // region could be empty string urlRegion = GetOptS3(urlWithOptions, "region"); // read configurations from file Config s3Cfg(configPath); S3_CHECK_OR_DIE(s3Cfg.Handle() != NULL, S3RuntimeError, "Failed to parse config file '" + configPath + "', or it doesn't exist"); S3_CHECK_OR_DIE(s3Cfg.SectionExist(configSection), S3ConfigError, "Selected section '" + configSection + "' does not exist, please check your configuration file", configSection); bool useHttps = s3Cfg.GetBool(configSection, "encryption", "true"); bool verifyCert = s3Cfg.GetBool(configSection, "verifycert", "true"); string version = s3Cfg.Get(configSection, "version", ""); S3Params params(sourceUrl, useHttps, version, urlRegion); string content = s3Cfg.Get(configSection, "loglevel", "WARNING"); s3ext_loglevel = getLogLevel(content.c_str()); content = s3Cfg.Get(configSection, "logtype", "INTERNAL"); s3ext_logtype = getLogType(content.c_str()); params.setDebugCurl(s3Cfg.GetBool(configSection, "debug_curl", "false")); params.setCred(s3Cfg.Get(configSection, "accessid", ""), s3Cfg.Get(configSection, "secret", ""), s3Cfg.Get(configSection, "token", "")); s3ext_logserverhost = s3Cfg.Get(configSection, "logserverhost", "127.0.0.1"); s3ext_logserverport = s3Cfg.SafeScan("logserverport", configSection, 1111, 1, 65535); int64_t numOfChunks = s3Cfg.SafeScan("threadnum", configSection, 4, 1, 8); params.setNumOfChunks(numOfChunks); int64_t chunkSize = s3Cfg.SafeScan("chunksize", configSection, 64 * 1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024); params.setChunkSize(chunkSize); int64_t lowSpeedLimit = s3Cfg.SafeScan("low_speed_limit", configSection, 10240, 0, INT_MAX); params.setLowSpeedLimit(lowSpeedLimit); int64_t lowSpeedTime = s3Cfg.SafeScan("low_speed_time", configSection, 60, 0, INT_MAX); params.setLowSpeedTime(lowSpeedTime); params.setVerifyCert(verifyCert); CheckEssentialConfig(params); return params; }