static Try<Nothing> validateUris(const CommandInfo& commandInfo) { foreach (const CommandInfo::URI& uri, commandInfo.uris()) { Try<Nothing> uriValidation = Fetcher::validateUri(uri.value()); if (uriValidation.isError()) { return Error(uriValidation.error()); } if (uri.has_output_file()) { Try<Nothing> outputFileValidation = Fetcher::validateOutputFile(uri.output_file()); if (outputFileValidation.isError()) { return Error(outputFileValidation.error()); } } } return Nothing(); }
Future<Nothing> Fetcher::fetch( const ContainerID& containerId, const CommandInfo& commandInfo, const string& sandboxDirectory, const Option<string>& user, const SlaveID& slaveId, const Flags& flags) { if (commandInfo.uris().size() == 0) { return Nothing(); } return dispatch(process.get(), &FetcherProcess::fetch, containerId, commandInfo, sandboxDirectory, user, slaveId, flags); }
bool operator == (const CommandInfo& left, const CommandInfo& right) { if (left.uris().size() != right.uris().size()) { return false; } // TODO(vinod): Factor out the comparison for repeated fields. for (int i = 0; i < left.uris().size(); i++) { bool found = false; for (int j = 0; j < right.uris().size(); j++) { if (left.uris().Get(i) == right.uris().Get(j)) { found = true; break; } } if (!found) { return false; } } if (left.arguments().size() != right.arguments().size()) { return false; } // The order of argv is important. for (int i = 0; i < left.arguments().size(); i++) { if (left.arguments().Get(i) != right.arguments().Get(i)) { return false; } } // NOTE: We are not validating CommandInfo::ContainerInfo here // because it is being deprecated in favor of ContainerInfo. // TODO(vinod): Kill the above comment when // CommandInfo::ContainerInfo is removed. return left.environment() == right.environment() && left.value() == right.value() && left.user() == right.user() && left.shell() == right.shell(); }
inline bool operator == (const CommandInfo& left, const CommandInfo& right) { if (left.uris().size() != right.uris().size()) { return false; } for (int i=0; i<left.uris().size(); i++) { bool found = false; for (int j=0; j<right.uris().size(); j++) { if (left.uris().Get(i) == right.uris().Get(j)) { found = true; break; } } if (!found) { return false; } } return left.has_environment() == right.has_environment() && (!left.has_environment() || (left.environment() == right.environment())) && left.value() == right.value(); }
Future<Nothing> FetcherProcess::fetch( const ContainerID& containerId, const CommandInfo& commandInfo, const string& sandboxDirectory, const Option<string>& user, const SlaveID& slaveId, const Flags& flags) { VLOG(1) << "Starting to fetch URIs for container: " << containerId << ", directory: " << sandboxDirectory; // TODO(bernd-mesos): This will disappear once we inject flags at // Fetcher/FetcherProcess creation time. For now we trust this is // always the exact same value. cache.setSpace(flags.fetcher_cache_size); Try<Nothing> validated = validateUris(commandInfo); if (validated.isError()) { return Failure("Could not fetch: " + validated.error()); } Option<string> commandUser = user; if (commandInfo.has_user()) { commandUser = commandInfo.user(); } string cacheDirectory = paths::getSlavePath(flags.fetcher_cache_dir, slaveId); if (commandUser.isSome()) { // Segregating per-user cache directories. cacheDirectory = path::join(cacheDirectory, commandUser.get()); } if (commandUser.isSome()) { // First assure that we are working for a valid user. // TODO(bernd-mesos): This should be asynchronous. Try<Nothing> chown = os::chown(commandUser.get(), sandboxDirectory); if (chown.isError()) { return Failure("Failed to chown directory: " + sandboxDirectory + " to user: "******" with error: " + chown.error()); } } // For each URI we determine if we should use the cache and if so we // try and either get the cache entry or create a cache entry. If // we're getting the cache entry then we might need to wait for that // cache entry to be downloaded. If we're creating a new cache entry // then we need to properly reserve the cache space (and perform any // evictions). Thus, there are three possibilities for each URI: // // (1) We are not using the cache. // (2) We are using the cache but need to wait for an entry to be // downloaded. // (3) We are using the cache and need to create a new entry. // // We capture whether or not we're using the cache using an Option // as a value in a map, i.e., if we are not trying to use the cache // as in (1) above then the Option is None otherwise as in (2) and // (3) the Option is Some. And to capture the asynchronous nature of // both (2) and (3) that Option holds a Future to the actual cache // entry. hashmap<CommandInfo::URI, Option<Future<shared_ptr<Cache::Entry>>>> entries; foreach (const CommandInfo::URI& uri, commandInfo.uris()) { if (!uri.cache()) { entries[uri] = None(); continue; } // Check if this is already in the cache (but not necessarily // downloaded). const Option<shared_ptr<Cache::Entry>> entry = cache.get(commandUser, uri.value()); if (entry.isSome()) { entry.get()->reference(); // Wait for the URI to be downloaded into the cache (or fail) entries[uri] = entry.get()->completion() .then(defer(self(), [=]() { return Future<shared_ptr<Cache::Entry>>(entry.get()); })); } else { shared_ptr<Cache::Entry> newEntry = cache.create(cacheDirectory, commandUser, uri); newEntry->reference(); entries[uri] = async([=]() { return fetchSize(uri.value(), flags.frameworks_home); }) .then(defer(self(), [=](const Try<Bytes>& requestedSpace) { return reserveCacheSpace(requestedSpace, newEntry); })); } } // NOTE: We explicitly call the continuation '_fetch' even though it // looks like we could easily inline it here because we want to be // able to mock the function for testing! Don't remove this! return _fetch(entries, containerId, sandboxDirectory, cacheDirectory, commandUser, flags); }
int main(int argc, char* argv[]) { GOOGLE_PROTOBUF_VERIFY_VERSION; CommandInfo commandInfo; // Construct URIs from the encoded environment string. const std::string& uris = os::getenv("MESOS_EXECUTOR_URIS"); foreach (const std::string& token, strings::tokenize(uris, " ")) { // Delimiter between URI, execute permission and extract options // Expected format: {URI}+[01][XN] // {URI} - The actual URI for the asset to fetch // [01] - 1 if the execute permission should be set else 0 // [XN] - X if we should extract the URI (if it's compressed) else N size_t pos = token.rfind("+"); CHECK(pos != std::string::npos) << "Invalid executor uri token in env " << token; CommandInfo::URI uri; uri.set_value(token.substr(0, pos)); uri.set_executable(token.substr(pos + 1, 1) == "1"); uri.set_extract(token.substr(pos + 2, 1) == "X"); commandInfo.add_uris()->MergeFrom(uri); } CHECK(os::hasenv("MESOS_WORK_DIRECTORY")) << "Missing MESOS_WORK_DIRECTORY environment variable"; std::string directory = os::getenv("MESOS_WORK_DIRECTORY"); // We cannot use Some in the ternary expression because the compiler needs to // be able to infer the type, thus the explicit Option<string>. // TODO(idownes): Add an os::hasenv that returns an Option<string>. Option<std::string> user = os::hasenv("MESOS_USER") ? Option<std::string>(os::getenv("MESOS_USER")) // Explicit so it compiles. : None(); // Fetch each URI to a local file, chmod, then chown if a user is provided. foreach (const CommandInfo::URI& uri, commandInfo.uris()) { // Fetch the URI to a local file. Try<string> fetched = fetch(uri.value(), directory); if (fetched.isError()) { EXIT(1) << "Failed to fetch: " << uri.value(); } // Chmod the fetched URI if it's executable, else assume it's an archive // that should be extracted. if (uri.executable()) { Try<Nothing> chmod = os::chmod( fetched.get(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); if (chmod.isError()) { EXIT(1) << "Failed to chmod " << fetched.get() << ": " << chmod.error(); } } else if (uri.extract()) { //TODO(idownes): Consider removing the archive once extracted. // Try to extract the file if it's recognized as an archive. Try<bool> extracted = extract(fetched.get(), directory); if (extracted.isError()) { EXIT(1) << "Failed to extract " << fetched.get() << ":" << extracted.error(); } } else { LOG(INFO) << "Skipped extracting path '" << fetched.get() << "'"; } // Recursively chown the directory if a user is provided. if (user.isSome()) { Try<Nothing> chowned = os::chown(user.get(), directory); if (chowned.isError()) { EXIT(1) << "Failed to chown " << directory << ": " << chowned.error(); } } } return 0; }