TEST_F(FilesystemTests, test_wildcard_single_file_list) {
std::vector<std::string> files;
std::vector<std::string> files_flag;
auto status = resolveFilePattern(kFakeDirectory + "/%", files);
auto status2 =
resolveFilePattern(kFakeDirectory + "/%", files_flag, REC_LIST_FILES);
EXPECT_TRUE(status.ok());
EXPECT_EQ(files.size(), 3);
EXPECT_EQ(files.size(), files_flag.size());
EXPECT_NE(std::find(files.begin(), files.end(), kFakeDirectory + "/roto.txt"),
files.end());
}
QueryData genSafariExtensions(QueryContext& context) {
QueryData results;
// Iterate over each user
auto users = usersFromContext(context);
for (const auto& row : users) {
if (row.count("uid") > 0 && row.count("directory") > 0) {
auto dir = fs::path(row.at("directory")) / kSafariExtensionsPath;
// Check that an extensions directory exists.
if (!pathExists(dir).ok()) {
continue;
}
// Glob the extension files.
std::vector<std::string> paths;
if (!resolveFilePattern(dir / kSafariExtensionsPattern, paths).ok()) {
continue;
}
for (const auto& extension_path : paths) {
genSafariExtension(row.at("uid"), extension_path, results);
}
}
}
return results;
}
std::set<std::string> FSEventsEventPublisher::transformSubscription(
FSEventsSubscriptionContextRef& sc) const {
std::set<std::string> paths;
sc->discovered_ = sc->path;
if (sc->path.find("**") != std::string::npos) {
// Double star will indicate recursive matches, restricted to endings.
sc->recursive = true;
sc->discovered_ = sc->path.substr(0, sc->path.find("**"));
// Remove '**' from the subscription path (used to match later).
sc->path = sc->discovered_;
}
// If the path 'still' OR 'either' contains a single wildcard.
if (sc->path.find('*') != std::string::npos) {
// First check if the wildcard is applied to the end.
auto fullpath = fs::path(sc->path);
if (fullpath.filename().string().find('*') != std::string::npos) {
sc->discovered_ = fullpath.parent_path().string();
}
// FSEvents needs a real path, if the wildcard is within the path then
// a configure-time resolve is required.
if (sc->discovered_.find('*') != std::string::npos) {
std::vector<std::string> exploded_paths;
resolveFilePattern(sc->discovered_, exploded_paths);
for (const auto& path : exploded_paths) {
paths.insert(path);
}
sc->recursive_match = sc->recursive;
return paths;
}
}
paths.insert(sc->discovered_);
return paths;
}
QueryData genChromeBasedExtensions(QueryContext& context, const fs::path sub_dir) {
QueryData results;
auto homes = osquery::getHomeDirectories();
for (const auto& home : homes) {
// For each user, enumerate all of their chrome profiles.
std::vector<std::string> profiles;
fs::path extension_path = home / sub_dir;
if (!resolveFilePattern(extension_path, profiles, REC_LIST_FOLDERS).ok()) {
continue;
}
// For each profile list each extension in the Extensions directory.
std::vector<std::string> extensions;
for (const auto& profile : profiles) {
listDirectoriesInDirectory(profile, extensions);
}
// Generate an addons list from their extensions JSON.
std::vector<std::string> versions;
for (const auto& extension : extensions) {
listDirectoriesInDirectory(extension, versions);
}
// Extensions use /<EXTENSION>/<VERSION>/manifest.json.
for (const auto& version : versions) {
genExtension(version, results);
}
}
return results;
}
int getBluetoothSharingStatus() {
auto users = SQL::selectAllFrom("users");
for (const auto& row : users) {
if (row.count("uid") > 0 && row.count("directory") > 0) {
auto dir = fs::path(row.at("directory")) / kRemoteBluetoothSharingPath;
if (!pathExists(dir).ok()) {
continue;
}
std::vector<std::string> paths;
if (!resolveFilePattern(dir / kRemoteBluetoothSharingPattern, paths)
.ok()) {
continue;
}
for (const auto& bluetoothSharing_path : paths) {
auto bluetoothSharingStatus =
SQL::selectAllFrom("plist", "path", EQUALS, bluetoothSharing_path);
if (bluetoothSharingStatus.empty()) {
continue;
}
for (const auto& r : bluetoothSharingStatus) {
if (r.find("key") == row.end() || row.find("value") == r.end()) {
continue;
}
if (r.at("key") == "PrefKeyServicesEnabled" &&
r.at("value") == INTEGER(1)) {
return 1;
}
}
}
}
}
return 0;
}
QueryData genChocolateyPackages(QueryContext& context) {
QueryData results;
auto chocoEnvInstall = getEnvVar("ChocolateyInstall");
fs::path chocoInstallPath;
if (chocoEnvInstall.is_initialized()) {
chocoInstallPath = fs::path(*chocoEnvInstall);
}
if (chocoInstallPath.empty()) {
LOG(WARNING) << "Did not find chocolatey path environment variable";
return results;
}
auto nuspecPattern = chocoInstallPath / "lib/%/%.nuspec";
std::vector<std::string> manifests;
resolveFilePattern(nuspecPattern, manifests, GLOB_FILES);
for (const auto& pkg : manifests) {
Row r;
auto s = genPackage(pkg, r);
if (!s.ok()) {
VLOG(1) << "Failed to parse " << pkg << " with " << s.getMessage();
}
results.push_back(r);
}
return results;
}
QueryData genChromeBasedExtensions(QueryContext& context,
const fs::path& sub_dir) {
QueryData results;
auto users = usersFromContext(context);
for (const auto& row : users) {
if (row.count("uid") > 0 && row.count("directory") > 0) {
// For each user, enumerate all of their chrome profiles.
std::vector<std::string> profiles;
fs::path extension_path = row.at("directory") / sub_dir;
if (!resolveFilePattern(extension_path, profiles, GLOB_FOLDERS).ok()) {
continue;
}
// For each profile list each extension in the Extensions directory.
std::vector<std::string> extensions;
for (const auto& profile : profiles) {
listDirectoriesInDirectory(profile, extensions);
}
// Generate an addons list from their extensions JSON.
std::vector<std::string> versions;
for (const auto& extension : extensions) {
listDirectoriesInDirectory(extension, versions);
}
// Extensions use /<EXTENSION>/<VERSION>/manifest.json.
for (const auto& version : versions) {
genExtension(row.at("uid"), version, results);
}
}
}
return results;
}
bool INotifyEventPublisher::monitorSubscription(
INotifySubscriptionContextRef& sc, bool add_watch) {
sc->discovered_ = sc->path;
if (sc->path.find("**") != std::string::npos) {
sc->recursive = true;
sc->discovered_ = sc->path.substr(0, sc->path.find("**"));
sc->path = sc->discovered_;
}
if (sc->path.find('*') != std::string::npos) {
// If the wildcard exists within the file (leaf), remove and monitor the
// directory instead. Apply a fnmatch on fired events to filter leafs.
auto fullpath = fs::path(sc->path);
if (fullpath.filename().string().find('*') != std::string::npos) {
sc->discovered_ = fullpath.parent_path().string() + '/';
}
if (sc->discovered_.find('*') != std::string::npos) {
// If a wildcard exists within the tree (stem), resolve at configure
// time and monitor each path.
std::vector<std::string> paths;
resolveFilePattern(sc->discovered_, paths);
for (const auto& _path : paths) {
addMonitor(_path, sc->mask, sc->recursive, add_watch);
}
sc->recursive_match = sc->recursive;
return true;
}
}
if (isDirectory(sc->discovered_) && sc->discovered_.back() != '/') {
sc->path += '/';
sc->discovered_ += '/';
}
return addMonitor(sc->discovered_, sc->mask, sc->recursive, add_watch);
}
inline void mergeAdditional(const tree_node& node, ConfigData& conf) {
if (conf.all_data.count("additional_monitoring") > 0) {
conf.all_data.get_child("additional_monitoring").erase(node.first);
}
conf.all_data.add_child("additional_monitoring." + node.first, node.second);
// Support special merging of file paths.
if (node.first == "file_paths") {
for (const auto& category : node.second) {
for (const auto& path : category.second) {
resolveFilePattern(path.second.data(),
conf.files[category.first],
REC_LIST_FOLDERS | REC_EVENT_OPT);
}
}
} else if (node.first == "yara") {
for (const auto& category : node.second) {
// Todo: the yara category key must come after "file_paths".
if (conf.files.find(category.first) == conf.files.end()) {
continue;
}
for (const auto& file : category.second) {
conf.yara[category.first].push_back(
file.second.get_value<std::string>());
}
}
} else {
// Unknown additional monitoring key.
}
}
inline void removeStalePaths(const std::string& manager) {
std::vector<std::string> paths;
// Attempt to remove all stale extension sockets.
resolveFilePattern(manager + ".*", paths);
for (const auto& path : paths) {
remove(path);
}
}
TEST_F(FilesystemTests, test_double_wild_event_opt) {
std::vector<std::string> all;
auto status = resolveFilePattern(
kFakeDirectory + "/%%", all, REC_LIST_FOLDERS | REC_EVENT_OPT);
EXPECT_TRUE(status.ok());
EXPECT_EQ(all.size(), 1);
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end());
}
TEST_F(FilesystemTests, test_wildcard_full_recursion) {
std::vector<std::string> files;
auto status = resolveFilePattern(kFakeDirectory + "/%%", files);
EXPECT_TRUE(status.ok());
EXPECT_NE(std::find(files.begin(),
files.end(),
kFakeDirectory + "/deep1/deep2/level2.txt"),
files.end());
}
TEST_F(FilesystemTests, test_wildcard_end_last_component) {
std::vector<std::string> files;
auto status = resolveFilePattern(kFakeDirectory + "/%11/%sh", files);
EXPECT_TRUE(status.ok());
EXPECT_NE(std::find(files.begin(),
files.end(),
kFakeDirectory + "/deep11/not_bash"),
files.end());
}
TEST_F(FilesystemTests, test_wildcard_three_kinds) {
std::vector<std::string> files;
auto status = resolveFilePattern(kFakeDirectory + "/%p11/%/%%", files);
EXPECT_TRUE(status.ok());
EXPECT_NE(std::find(files.begin(),
files.end(),
kFakeDirectory + "/deep11/deep2/deep3/level3.txt"),
files.end());
}
TEST_F(FilesystemTests, test_no_wild) {
std::vector<std::string> all;
auto status =
resolveFilePattern(kFakeDirectory + "/roto.txt", all, REC_LIST_FILES);
EXPECT_TRUE(status.ok());
EXPECT_EQ(all.size(), 1);
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
all.end());
}
inline void mergeFilePath(const std::string& name,
const tree_node& node,
ConfigData& conf) {
for (const auto& path : node.second) {
resolveFilePattern(path.second.data(),
conf.files[node.first],
REC_LIST_FOLDERS | REC_EVENT_OPT);
}
conf.all_data.add_child(name + "." + node.first, node.second);
}
TEST_F(FilesystemTests, test_wildcard_single_all_list) {
std::vector<std::string> all;
auto status = resolveFilePattern(kFakeDirectory + "/%", all, REC_LIST_ALL);
EXPECT_TRUE(status.ok());
EXPECT_EQ(all.size(), 6);
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
all.end());
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep11"),
all.end());
}
TEST_F(FilesystemTests, test_wildcard_single_folder_list) {
std::vector<std::string> folders;
auto status =
resolveFilePattern(kFakeDirectory + "/%", folders, REC_LIST_FOLDERS);
EXPECT_TRUE(status.ok());
EXPECT_EQ(folders.size(), 3);
EXPECT_NE(
std::find(folders.begin(), folders.end(), kFakeDirectory + "/deep11"),
folders.end());
}
TEST_F(FilesystemTests, test_dotdot) {
std::vector<std::string> all;
auto status = resolveFilePattern(
kFakeDirectory + "/deep11/deep2/../../%", all, REC_LIST_FILES);
EXPECT_TRUE(status.ok());
EXPECT_EQ(all.size(), 3);
EXPECT_NE(std::find(all.begin(),
all.end(),
kFakeDirectory + "/deep11/deep2/../../door.txt"),
all.end());
}
TEST_F(FilesystemTests, test_letter_wild_opt) {
std::vector<std::string> all;
auto status = resolveFilePattern(
kFakeDirectory + "/d%", all, REC_LIST_FOLDERS | REC_EVENT_OPT);
EXPECT_TRUE(status.ok());
EXPECT_EQ(all.size(), 3);
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep1"),
all.end());
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/door.txt"),
all.end());
}
TEST_F(FilesystemTests, test_wildcard_double_folders) {
std::vector<std::string> all;
auto status =
resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_FOLDERS);
EXPECT_TRUE(status.ok());
EXPECT_EQ(all.size(), 6);
EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end());
EXPECT_NE(
std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"),
all.end());
}
TEST_F(FilesystemTests, test_dotdot_relative) {
std::vector<std::string> all;
auto status = resolveFilePattern(kTestDataPath + "%", all, REC_LIST_ALL);
EXPECT_TRUE(status.ok());
bool found = false;
for (const auto& file : all) {
if (file.find("test.config")) {
found = true;
break;
}
}
EXPECT_TRUE(found);
}
Status FilesystemConfigPlugin::genPack(const std::string& name,
const std::string& value,
std::string& pack) {
if (name == "*") {
// The config requested a multi-pack.
std::vector<std::string> paths;
resolveFilePattern(value, paths);
pt::ptree multi_pack;
for (const auto& path : paths) {
std::string content;
if (!readFile(path, content)) {
LOG(WARNING) << "Cannot read multi-pack file: " << path;
continue;
}
// Assemble an intermediate property tree for simplified parsing.
pt::ptree single_pack;
stripConfigComments(content);
try {
std::stringstream json_stream;
json_stream << content;
pt::read_json(json_stream, single_pack);
} catch (const pt::json_parser::json_parser_error& /* e */) {
LOG(WARNING) << "Cannot read multi-pack JSON: " << path;
continue;
}
multi_pack.put_child(fs::path(path).stem().string(), single_pack);
}
// We should have a property tree of pack content mimicking embedded
// configuration packs, ready to parse as a string.
std::ostringstream output;
pt::write_json(output, multi_pack, false);
pack = output.str();
if (pack.empty()) {
return Status(1, "Multi-pack content empty");
}
return Status(0);
}
boost::system::error_code ec;
if (!fs::is_regular_file(value, ec) || ec.value() != errc::success) {
return Status(1, value + " is not a valid path");
}
return readFile(value, pack);
}
QueryData genQuicklookCache(QueryContext& context) {
QueryData results;
// There may be several quick look caches.
// Apply a GLOB search since the folder is randomized.
std::vector<std::string> databases;
if (!resolveFilePattern(kQuicklookPattern, databases)) {
return results;
}
for (const auto& index : databases) {
sqlite3* db = nullptr;
auto rc = sqlite3_open_v2(
index.c_str(), &db,
(SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE | SQLITE_OPEN_NOMUTEX),
nullptr);
if (rc != SQLITE_OK || db == nullptr) {
VLOG(1) << "Cannot open " << index << " read only: "
<< rc << " " << getStringForSQLiteReturnCode(rc);
if (db != nullptr) {
sqlite3_close(db);
}
continue;
}
// QueryData file_results;
std::string query =
"SELECT f.*, last_hit_date, hit_count, icon_mode FROM (SELECT rowid, * "
"FROM files) f, (SELECT *, max(last_hit_date) AS last_hit_date FROM "
"thumbnails GROUP BY file_id) t WHERE t.file_id = rowid;";
sqlite3_stmt* stmt = nullptr;
rc = sqlite3_prepare_v2(db, query.c_str(), -1, &stmt, nullptr);
while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
Row r;
genQuicklookRow(stmt, r);
// For each row added to the results from this database, add the path to
// the database, then move into the table's result set.
r["cache_path"] = index;
results.push_back(r);
}
// Clean up.
sqlite3_finalize(stmt);
sqlite3_close(db);
}
return results;
}
QueryData ATCPlugin::generate(QueryContext& context) {
QueryData qd;
std::vector<std::string> paths;
auto s = resolveFilePattern(path_, paths);
if (!s.ok()) {
LOG(WARNING) << "Could not glob: " << path_;
}
for (const auto& path : paths) {
s = genQueryDataForSqliteTable(path, sqlite_query_, qd, false);
if (!s.ok()) {
LOG(WARNING) << "Error Code: " << s.getCode()
<< " Could not generate data: " << s.getMessage();
}
}
return qd;
}
QueryData genOSXPlist(QueryContext& context) {
QueryData results;
// Resolve file paths for EQUALS and LIKE operations.
auto paths = context.constraints["path"].getAll(EQUALS);
context.expandConstraints(
"path",
LIKE,
paths,
([&](const std::string& pattern, std::set<std::string>& out) {
std::vector<std::string> patterns;
auto status =
resolveFilePattern(pattern, patterns, GLOB_ALL | GLOB_NO_CANON);
if (status.ok()) {
for (const auto& resolved : patterns) {
out.insert(resolved);
}
}
return status;
}));
for (const auto& path : paths) {
if (!pathExists(path).ok() || !isReadable(path).ok()) {
VLOG(1) << "Cannot find/read defaults plist from path: " + path;
continue;
}
pt::ptree tree;
if (!osquery::parsePlist(path, tree).ok()) {
VLOG(1) << "Could not parse plist: " + path;
continue;
}
for (const auto& item : tree) {
Row r;
r["path"] = path;
r["key"] = item.first;
r["subkey"] = "";
genOSXPlistPrefValue(item.second, r, 0, results);
}
}
return results;
}
Status FilesystemConfigPlugin::genConfig(
std::map<std::string, std::string>& config) {
if (!fs::is_regular_file(FLAGS_config_path)) {
return Status(1, "config file does not exist");
}
std::vector<std::string> conf_files;
resolveFilePattern(FLAGS_config_path + ".d/%.conf", conf_files);
std::sort(conf_files.begin(), conf_files.end());
conf_files.push_back(FLAGS_config_path);
for (const auto& path : conf_files) {
std::string content;
if (readFile(path, content).ok()) {
config[path] = content;
}
}
return Status(0, "OK");
}
QueryData genSystemControls(QueryContext& context) {
QueryData results;
// Read the sysctl.conf values.
std::map<std::string, std::string> config;
for (const auto& path : kControlSettingsFiles) {
genControlConfigFromPath(path, config);
}
for (const auto& dirs : kControlSettingsDirs) {
std::vector<std::string> configs;
if (resolveFilePattern(dirs, configs).ok()) {
for (const auto& path : configs) {
genControlConfigFromPath(path, config);
}
}
}
// Iterate through the sysctl-defined macro of control types.
if (context.constraints["name"].exists(EQUALS)) {
// Request MIB information by the description (name).
auto names = context.constraints["name"].getAll(EQUALS);
for (const auto& name : names) {
genControlInfoFromName(name, results, config);
}
} else if (context.constraints["oid"].exists(EQUALS)) {
// Request MIB by OID as a string, parse into set of INTs.
auto oids = context.constraints["oid"].getAll(EQUALS);
for (const auto& oid_string : oids) {
genControlInfoFromOIDString(oid_string, results, config);
}
} else if (context.constraints["subsystem"].exists(EQUALS)) {
// Limit the MIB search to a subsystem name (first find the INT).
auto subsystems = context.constraints["subsystem"].getAll(EQUALS);
for (const auto& subsystem : subsystems) {
genAllControls(results, config, subsystem);
}
} else {
genAllControls(results, config, "");
}
return results;
}
void expandFSPathConstraints(QueryContext& context,
const std::string& path_column_name,
std::set<std::string>& paths) {
context.expandConstraints(
path_column_name,
LIKE,
paths,
([&](const std::string& pattern, std::set<std::string>& out) {
std::vector<std::string> patterns;
auto status =
resolveFilePattern(pattern, patterns, GLOB_ALL | GLOB_NO_CANON);
if (status.ok()) {
for (const auto& resolved : patterns) {
out.insert(resolved);
}
}
return status;
}));
}
void INotifyEventPublisher::configure() {
for (auto& sub : subscriptions_) {
// Anytime a configure is called, try to monitor all subscriptions.
// Configure is called as a response to removing/adding subscriptions.
// This means recalculating all monitored paths.
auto sc = getSubscriptionContext(sub->context);
if (sc->discovered_.size() > 0) {
continue;
}
sc->discovered_ = sc->path;
if (sc->path.find("**") != std::string::npos) {
sc->recursive = true;
sc->discovered_ = sc->path.substr(0, sc->path.find("**"));
sc->path = sc->discovered_;
}
if (sc->path.find('*') != std::string::npos) {
// If the wildcard exists within the file (leaf), remove and monitor the
// directory instead. Apply a fnmatch on fired events to filter leafs.
auto fullpath = fs::path(sc->path);
if (fullpath.filename().string().find('*') != std::string::npos) {
sc->discovered_ = fullpath.parent_path().string();
}
if (sc->discovered_.find('*') != std::string::npos) {
// If a wildcard exists within the tree (stem), resolve at configure
// time and monitor each path.
std::vector<std::string> paths;
resolveFilePattern(sc->discovered_, paths);
for (const auto& _path : paths) {
addMonitor(_path, sc->recursive);
}
sc->recursive_match = sc->recursive;
continue;
}
}
addMonitor(sc->discovered_, sc->recursive);
}
}
