// update a file's manifest, in response to a remote call
// return 0 on success
// return -ENOENT if not found
// return -ESTALE if not local
// return -errno on error
// NOTE: the permissions will already have been checked by the server
static int UG_impl_manifest_patch( struct SG_gateway* gateway, struct SG_request_data* reqdat, struct SG_manifest* write_delta, void* cls ) {
int rc = 0;
int ref_rc = 0;
struct fskit_entry* fent = NULL;
struct UG_state* ug = (struct UG_state*)SG_gateway_cls( gateway );
struct fskit_core* fs = UG_state_fs( ug );
struct UG_inode* inode = NULL;
struct ms_client* ms = SG_gateway_ms( gateway );
uint64_t volume_id = ms_client_get_volume_id( ms );
rc = UG_consistency_path_ensure_fresh( gateway, reqdat->fs_path );
if( rc != 0 ) {
SG_error("UG_consistency_path_ensure_fresh('%s') rc = %d\n", reqdat->fs_path, rc );
return rc;
}
rc = UG_consistency_manifest_ensure_fresh( gateway, reqdat->fs_path );
if( rc != 0 ) {
SG_error("UG_consistency_manifest_ensure_fresh('%s') rc = %d\n", reqdat->fs_path, rc );
return rc;
}
// look up
fent = fskit_entry_resolve_path( fs, reqdat->fs_path, reqdat->user_id, volume_id, true, &rc );
if( fent == NULL ) {
return rc;
}
inode = (struct UG_inode*)fskit_entry_get_user_data( fent );
// must be coordinated by us
if( UG_inode_coordinator_id( inode ) != SG_gateway_id( gateway ) ) {
fskit_entry_unlock( fent );
return -ESTALE;
}
// update the manifest
fskit_entry_ref_entry( fent );
rc = UG_write_patch_manifest( gateway, reqdat, inode, write_delta );
fskit_entry_unlock( fent );
ref_rc = fskit_entry_unref( fs, reqdat->fs_path, fent );
if( ref_rc != 0 ) {
SG_warn("fskit_entry_unref('%s') rc = %d\n", reqdat->fs_path, rc );
}
return rc;
}
// truncate a file
// return 0 on success
// return -errno on error
static int UG_impl_truncate( struct SG_gateway* gateway, struct SG_request_data* reqdat, uint64_t new_size, void* cls ) {
int rc = 0;
struct UG_state* ug = (struct UG_state*)SG_gateway_cls( gateway );
struct fskit_core* fs = UG_state_fs( ug );
struct ms_client* ms = (struct ms_client*)SG_gateway_ms( gateway );
uint64_t volume_id = ms_client_get_volume_id( ms );
// truncate locally. The MS will be informed as part of the user route.
rc = fskit_trunc( fs, reqdat->fs_path, reqdat->user_id, volume_id, new_size );
if( rc != 0 ) {
SG_error("fskit_trunc( '%s', %" PRIu64 ") rc = %d\n", reqdat->fs_path, new_size, rc);
}
return rc;
}
// detach a file
// return 0 on success
// return -errno on error
static int UG_impl_detach( struct SG_gateway* gateway, struct SG_request_data* reqdat, void* cls ) {
int rc = 0;
struct UG_state* ug = (struct UG_state*)SG_gateway_cls( gateway );
struct fskit_core* fs = UG_state_fs( ug );
struct ms_client* ms = (struct ms_client*)SG_gateway_ms( gateway );
uint64_t volume_id = ms_client_get_volume_id( ms );
struct stat sb;
char const* method = NULL;
// file or directory?
rc = fskit_stat( fs, reqdat->fs_path, 0, 0, &sb );
if( rc != 0 ) {
return rc;
}
if( S_ISREG( sb.st_mode ) ) {
// unlink locally. The MS will be informed as part of the user route.
method = "fskit_unlink";
rc = fskit_unlink( fs, reqdat->fs_path, reqdat->user_id, volume_id );
}
else {
// rmdir locally. The MS will be informed as part of the user route
method = "fskit_rmdir";
rc = fskit_rmdir( fs, reqdat->fs_path, reqdat->user_id, volume_id );
}
if( rc != 0 ) {
SG_error("%s( '%s' ) rc = %d\n", method, reqdat->fs_path, rc);
}
return 0;
}
// populate and sign a manifest from the driver's published dataset info
// NOTE: the mi's cached metadata must be present
int AG_populate_manifest( Serialization::ManifestMsg* mmsg, char const* path, struct AG_map_info* mi, struct AG_driver_publish_info* pub_info ) {
// this only works for files...
if( mi->type != MD_ENTRY_FILE ) {
return -EINVAL;
}
// need cached MS metadata
if( !mi->cache_valid ) {
SG_error("Entry for %s does not have all cached metadata\n", path );
return -EINVAL;
}
struct AG_state* state = AG_get_state();
if( state == NULL ) {
// shutting down
return -ENOTCONN;
}
uint64_t volume_id = ms_client_get_volume_id( state->ms );
uint64_t gateway_id = state->ms->gateway_id;
uint64_t owner_id = state->ms->owner_id;
uint64_t block_size = ms_client_get_volume_blocksize( state->ms );
mmsg->set_volume_id( volume_id );
mmsg->set_coordinator_id( gateway_id );
mmsg->set_owner_id( owner_id );
mmsg->set_file_id( mi->file_id );
mmsg->set_file_version( mi->file_version );
// driver supplies these fields
mmsg->set_size( pub_info->size );
mmsg->set_fent_mtime_sec( pub_info->mtime_sec );
mmsg->set_fent_mtime_nsec( pub_info->mtime_nsec );
mmsg->set_mtime_sec( pub_info->mtime_sec );
mmsg->set_mtime_nsec( pub_info->mtime_nsec );
// calculate blockinfo
uint64_t num_blocks = 0;
if( pub_info->size >= 0 ) {
num_blocks = pub_info->size / block_size;
if( pub_info->size % block_size != 0 ) {
num_blocks++;
}
}
Serialization::BlockURLSetMsg *bbmsg = mmsg->add_block_url_set();
bbmsg->set_start_id( 0 );
bbmsg->set_end_id( num_blocks );
bbmsg->set_gateway_id( gateway_id );
// use the random block version that corresponds to this file
for( uint64_t i = 0; i < num_blocks; i++ ) {
bbmsg->add_block_versions( mi->block_version );
}
SG_debug("Manifest: volume=%" PRIu64 " coordinator=%" PRIu64 " owner=%" PRIu64 " file_id=%" PRIX64 " file_version=%" PRId64 " size=%zu mtime=%" PRId64 ".%" PRId32 " num_blocks=%" PRIu64 " block_version=%" PRId64 "\n",
volume_id, gateway_id, owner_id, mi->file_id, mi->file_version, pub_info->size, pub_info->mtime_sec, pub_info->mtime_nsec, num_blocks, mi->block_version );
// NOTE: no hashes, since they're served with the blocks directly (along with a signature)
// sign the message
int rc = md_sign< Serialization::ManifestMsg >( state->ms->gateway_key, mmsg );
AG_release_state( state );
if( rc != 0 ) {
SG_error("gateway_sign_manifest rc = %d\n", rc );
return rc;
}
return 0;
}
// begin downloading metadata for a directory.
// if least_unknown_generation >= 0, then use the generation number to generate the URL
// otherwise, use the batch (page) number
// return 0 on success
// return -ENOMEM on OOM
// return negative on failure to initialize or start the download
static int ms_client_get_dir_metadata_begin( struct ms_client* client, uint64_t parent_id, int64_t least_unknown_generation, int64_t batch_id, struct md_download_loop* dlloop, struct md_download_context* dlctx ) {
int rc = 0;
CURL* curl = NULL;
char* url = NULL;
char* auth_header = NULL;
uint64_t volume_id = ms_client_get_volume_id( client );
struct ms_client_get_dir_download_state* dlstate = NULL;
if( least_unknown_generation > 0 ) {
// least unknown generation
url = ms_client_file_listdir_url( client->url, volume_id, ms_client_volume_version( client ), ms_client_cert_version( client ), parent_id, -1, least_unknown_generation );
}
else {
// page id
url = ms_client_file_listdir_url( client->url, volume_id, ms_client_volume_version( client ), ms_client_cert_version( client ), parent_id, batch_id, -1 );
}
if( url == NULL ) {
return -ENOMEM;
}
// set up download state
dlstate = SG_CALLOC( struct ms_client_get_dir_download_state, 1 );
if( dlstate == NULL ) {
SG_safe_free( url );
return -ENOMEM;
}
// set up CURL
// TODO: connection pool
curl = curl_easy_init();
if( curl == NULL ) {
SG_safe_free( dlstate );
SG_safe_free( url );
return -ENOMEM;
}
// generate auth header
rc = ms_client_auth_header( client, url, &auth_header );
if( rc != 0 ) {
// failed!
curl_easy_cleanup( curl );
SG_safe_free( url );
SG_safe_free( dlstate );
return -ENOMEM;
}
ms_client_init_curl_handle( client, curl, url, auth_header );
// set up download
rc = md_download_context_init( dlctx, curl, MS_MAX_MSG_SIZE, dlstate );
if( rc != 0 ) {
SG_safe_free( dlstate );
SG_safe_free( url );
SG_safe_free( auth_header );
curl_easy_cleanup( curl );
return rc;
}
// watch the download
rc = md_download_loop_watch( dlloop, dlctx );
if( rc != 0 ) {
SG_error("md_download_loop_watch rc = %d\n", rc );
md_download_context_free( dlctx, NULL );
SG_safe_free( dlstate );
SG_safe_free( url );
SG_safe_free( auth_header );
curl_easy_cleanup( curl );
return rc;
}
// set up download state
ms_client_get_dir_download_state_init( dlstate, batch_id, url, auth_header );
// start download
rc = md_download_context_start( client->dl, dlctx );
if( rc != 0 ) {
md_download_context_free( dlctx, NULL );
ms_client_get_dir_download_state_free( dlstate );
dlstate = NULL;
curl_easy_cleanup( curl );
return rc;
}
return rc;
}
int main( int argc, char** argv ) {
int rc = 0;
struct md_opts opts;
int new_optind = 0;
uint64_t remote_gateway_id = 0;
uint64_t volume_id = 0;
uint64_t file_id = 0x1234567890ABCDEF;
int64_t file_version = 1234567890;
struct timespec ts;
uint64_t new_size = 0;
clock_gettime( CLOCK_REALTIME, &ts );
// make it every 20 seconds, to test caching
ts.tv_sec = (ts.tv_sec / 20) * 20;
ts.tv_nsec = 0;
struct ms_client* ms = NULL;
SG_messages::Request request;
SG_messages::Reply reply;
char* tmp = NULL;
char* gateway_id_str = NULL;
char* fs_path = NULL;
struct SG_request_data reqdat;
// read opts, and find the end of the syndicate options
rc = common_parse_opts( &opts, argc, argv, &new_optind );
if( rc != 0 ) {
usage( argv[0] );
}
md_opts_free( &opts );
// need a gateway ID and path
if( new_optind + 3 > argc ) {
printf("new_optind = %d, argc = %d\n", new_optind, argc );
usage( argv[0] );
}
gateway_id_str = argv[new_optind];
fs_path = argv[new_optind+1];
new_size = strtoull( argv[new_optind+2], &tmp, 10 );
if( *tmp != '\0' ) {
usage( argv[0] );
}
remote_gateway_id = strtoull( gateway_id_str, &tmp, 10 );
if( *tmp != '\0' ) {
usage( argv[0] );
}
// us
struct SG_gateway gateway;
memset( &gateway, 0, sizeof(struct SG_gateway) );
// start up
rc = SG_gateway_init( &gateway, SYNDICATE_UG, false, argc, argv );
if( rc != 0 ) {
SG_error("SG_gateway_init rc = %d\n", rc );
exit(1);
}
SG_info("%s", "Initialized\n");
ms = SG_gateway_ms( &gateway );
volume_id = ms_client_get_volume_id( ms );
SG_request_data_init( &reqdat );
reqdat.volume_id = volume_id;
reqdat.coordinator_id = remote_gateway_id;
reqdat.file_id = file_id;
reqdat.file_version = file_version;
reqdat.fs_path = fs_path;
// generate the request
rc = SG_client_request_TRUNCATE_setup( &gateway, &request, &reqdat, new_size );
if( rc != 0 ) {
SG_error("SG_client_request_TRUNCATE_setup rc = %d\n", rc );
exit(2);
}
common_print_request( &request );
// send it off
rc = SG_client_request_send( &gateway, remote_gateway_id, &request, NULL, &reply );
if( rc != 0 ) {
SG_error("SG_client_request_send rc = %d\n", rc );
exit(2);
}
// got a reply!
// print it out
printf("\n");
common_print_reply( &reply );
SG_gateway_shutdown( &gateway );
return 0;
}
int main( int argc, char** argv ) {
int rc = 0;
struct syndicate_state state;
struct md_opts opts;
struct UG_opts ug_opts;
int local_optind = 0;
uint64_t volume_id = 0;
ms_path_t path;
struct ms_client_multi_result result;
md_opts_default( &opts );
// get options
rc = md_opts_parse( &opts, argc, argv, &local_optind, NULL, NULL );
if( rc != 0 ) {
SG_error("md_opts_parse rc = %d\n", rc );
md_common_usage( argv[0] );
exit(1);
}
memset( &ug_opts, 0, sizeof(struct UG_opts) );
// connect to syndicate
rc = syndicate_client_init( &state, &opts, &ug_opts );
if( rc != 0 ) {
SG_error("syndicate_client_init rc = %d\n", rc );
exit(1);
}
// get volume ID
volume_id = ms_client_get_volume_id( state.ms );
printf("\n\n\nBegin getattr multi\n\n\n");
// get each path and file ID
for( int i = local_optind; i < argc; i++ ) {
struct ms_path_ent path_ent;
uint64_t file_id = 0;
// file ID
rc = sscanf( argv[i], "%" PRIX64, &file_id );
if( rc != 1 ) {
SG_error("failed to parse file_id ID '%s'\n", argv[i] );
exit(1);
}
printf(" getattr(%" PRIX64 ")\n", file_id );
ms_client_make_path_ent( &path_ent, volume_id, 0, file_id, 0, 0, 0, 0, NULL, NULL );
path.push_back( path_ent );
}
printf("\n\n\n");
// get all
rc = ms_client_getattr_multi( state.ms, &path, &result );
if( rc != 0 ) {
SG_error("ms_client_getattr_multi rc = %d\n", rc );
exit(1);
}
printf("\n\n\n");
for( unsigned int i = 0; i < result.num_ents; i++ ) {
if( result.ents[i].file_id != 0 ) {
printf("Entry: %" PRIX64 " %s mode=%o version=%" PRId64 " write_nonce=%" PRId64 " generation=%d\n",
result.ents[i].file_id, result.ents[i].name, result.ents[i].mode, result.ents[i].version, result.ents[i].write_nonce, result.ents[i].generation );
}
}
ms_client_multi_result_free( &result );
printf("\n\n\nEnd getattr multi\n\n\n");
syndicate_client_shutdown( &state, 0 );
return 0;
}
int main( int argc, char** argv ) {
int rc = 0;
struct md_opts opts;
int new_optind = 0;
uint64_t remote_gateway_id = 0;
uint64_t volume_id = 0;
uint64_t file_id = 0x1234567890ABCDEF;
int64_t file_version = 1234567890;
struct timespec ts;
uint64_t block_id = 0;
int64_t block_version = 0;
unsigned char fake_hash[32];
clock_gettime( CLOCK_REALTIME, &ts );
// make it every 20 seconds, to test caching
ts.tv_sec = (ts.tv_sec / 20) * 20;
ts.tv_nsec = 0;
struct ms_client* ms = NULL;
struct SG_manifest write_delta;
SG_messages::Request request;
SG_messages::Reply reply;
char* tmp = NULL;
char* gateway_id_str = NULL;
char* fs_path = NULL;
// read opts, and find the end of the syndicate options
rc = common_parse_opts( &opts, argc, argv, &new_optind );
if( rc != 0 ) {
usage( argv[0] );
}
md_opts_free( &opts );
// must have an even number of remaining options, beyond the file path and gateway id
if( (argc - new_optind) % 2 != 0 || new_optind + 2 >= argc ) {
printf("new_optind = %d, argc = %d\n", new_optind, argc );
usage( argv[0] );
}
gateway_id_str = argv[new_optind];
fs_path = argv[new_optind+1];
remote_gateway_id = strtoull( gateway_id_str, &tmp, 10 );
if( *tmp != '\0' ) {
usage( argv[0] );
}
// us
struct SG_gateway gateway;
memset( &gateway, 0, sizeof(struct SG_gateway) );
// start up
rc = SG_gateway_init( &gateway, SYNDICATE_UG, false, argc, argv );
if( rc != 0 ) {
SG_error("SG_gateway_init rc = %d\n", rc );
exit(1);
}
SG_info("%s", "Initialized\n");
ms = SG_gateway_ms( &gateway );
volume_id = ms_client_get_volume_id( ms );
// create write request
rc = SG_manifest_init( &write_delta, volume_id, remote_gateway_id, file_id, file_version );
if( rc != 0 ) {
SG_error("SG_manifst_init( write_delta ) rc = %d\n", rc );
exit(2);
}
// give it a modtime of mod-20 seconds
SG_manifest_set_modtime( &write_delta, ts.tv_sec, ts.tv_nsec );
// coordinator is the remote gateway
SG_manifest_set_coordinator_id( &write_delta, remote_gateway_id );
// fake hash...
for( int i = 0; i < 32; i++ ) {
fake_hash[i] = i;
}
// populate from argv
for( int i = new_optind + 2; i < argc; i += 2 ) {
block_id = strtoull( argv[i], &tmp, 10 );
if( *tmp != '\0' ) {
SG_error("Failed to parse block ID '%s'\n", argv[i] );
usage( argv[0] );
}
block_version = strtoll( argv[i+1], &tmp, 10 );
if( *tmp != '\0' ) {
SG_error("Failed to parse block version '%s'\n", argv[i] );
usage( argv[0] );
}
// make a block
struct SG_manifest_block block;
rc = SG_manifest_block_init( &block, block_id, block_version, fake_hash, 32 );
if( rc != 0 ) {
SG_error("SG_manifest_block_init rc = %d\n", rc );
exit(2);
}
// put this write
rc = SG_manifest_put_block( &write_delta, &block, true );
if( rc != 0 ) {
SG_error("SG_manifest_put_block rc = %d\n", rc );
exit(2);
}
SG_manifest_block_free( &block );
}
// generate the request
rc = SG_client_request_WRITE_setup( &gateway, &request, fs_path, &write_delta );
if( rc != 0 ) {
SG_error("SG_client_request_WRITE_setup rc = %d\n", rc );
exit(2);
}
common_print_request( &request );
// send it off
rc = SG_client_request_send( &gateway, remote_gateway_id, &request, NULL, &reply );
if( rc != 0 ) {
SG_error("SG_client_request_send rc = %d\n", rc );
exit(2);
}
// got a reply!
// print it out
printf("\n");
common_print_reply( &reply );
SG_gateway_shutdown( &gateway );
return 0;
}