/* * globus_l_module_initialize() */ static void globus_l_module_initialize() { /* * Initialize the threads package (can't use the standard interface since * it depends on threads) */ globus_i_thread_pre_activate(); globus_i_memory_pre_activate(); /* * Initialize the registered module table and list */ globus_hashtable_init(&globus_l_module_table, GLOBUS_L_MODULE_TABLE_SIZE, globus_hashtable_voidp_hash, globus_hashtable_voidp_keyeq); globus_l_module_list = GLOBUS_NULL; /* * Initialize the recursive mutex */ globus_l_module_mutex_init(&globus_l_module_mutex); globus_thread_key_create(&globus_l_activate_parent_key, NULL); globus_thread_key_create(&globus_l_deactivate_parent_key, NULL); /* * Now finish initializing the threads package */ globus_module_activate(GLOBUS_THREAD_MODULE); }
static int s_error_cache_init (void) { char * tmp_string; if(globus_module_activate(GLOBUS_OBJECT_MODULE) != GLOBUS_SUCCESS) { return GLOBUS_FAILURE; } globus_thread_key_create(&s_peek_key, s_key_destructor_func); globus_object_cache_init (&s_result_to_object_mapper); local_mutex_init (&s_result_to_object_mutex, NULL); s_next_available_result_count = 1; s_error_cache_initialized = 1; tmp_string = globus_module_getenv("GLOBUS_ERROR_OUTPUT"); if(tmp_string != GLOBUS_NULL) { globus_i_error_output = GLOBUS_TRUE; } tmp_string = globus_module_getenv("GLOBUS_ERROR_VERBOSE"); if(tmp_string != GLOBUS_NULL) { globus_i_error_verbose = GLOBUS_TRUE; } globus_thread_key_create(&globus_i_error_verbose_key, GLOBUS_NULL); return GLOBUS_SUCCESS; }
void thread_ids_init() { thread_ids_destruct_cnt = 0; globus_thread_key_create(&thread_ids, thread_ids_destruct); # if (DEBUG_LEVEL > 0) { globus_stdio_lock(); { printf("----: thread_ids_init() - complete\n"); } globus_stdio_unlock(); } # endif }
int main( int argc, char ** argv) { int rc; globus_gram_job_manager_config_t config; globus_gram_job_manager_t manager; char * sleeptime_str; long sleeptime = 0; globus_bool_t debug_mode_service = GLOBUS_FALSE; globus_bool_t located_active_jm = GLOBUS_FALSE; int http_body_fd = -1; int context_fd = -1; gss_cred_id_t cred = GSS_C_NO_CREDENTIAL; OM_uint32 major_status, minor_status; pid_t forked_starter = 0; globus_bool_t cgi_invoked = GLOBUS_FALSE; int lock_tries_left = 10; if ((sleeptime_str = getenv("GLOBUS_JOB_MANAGER_SLEEP"))) { sleeptime = atoi(sleeptime_str); sleep(sleeptime); } if (getenv("GATEWAY_INTERFACE")) { cgi_invoked = GLOBUS_TRUE; } /* * Stdin and stdout point at socket to client * Make sure no buffering. * stderr may also, depending on the option in the grid-services */ setbuf(stdout,NULL); /* Don't export these to the perl scripts */ fcntl(STDIN_FILENO, F_SETFD, (int) 1); fcntl(STDOUT_FILENO, F_SETFD, (int) 1); fcntl(STDERR_FILENO, F_SETFD, (int) 1); /* * At least have minimal POSIX path for job environment via extra * environment values */ if(getenv("PATH") == NULL) { char * path; char default_path[] = "/usr/bin:/bin"; size_t pathlen; pathlen = confstr(_CS_PATH, NULL, (size_t) 0); if (pathlen < sizeof(default_path)) { pathlen = sizeof(default_path); } path = malloc(pathlen); path[0] = 0; (void) confstr(_CS_PATH, path, pathlen); if (path[0] == 0) { strncpy(path, default_path, pathlen); } setenv("PATH", path, 1); } /* Force non-threaded execution for now */ globus_thread_set_model(GLOBUS_THREAD_MODEL_NONE); /* Activate a common before parsing command-line so that * things work. Note that we can't activate everything yet because we might * set the GLOBUS_TCP_PORT_RANGE after parsing command-line args and we * need that set before activating XIO. */ rc = globus_module_activate(GLOBUS_COMMON_MODULE); if (rc != GLOBUS_SUCCESS) { fprintf(stderr, "Error activating GLOBUS_COMMON_MODULE\n"); exit(1); } /* Parse command line options to get jobmanager configuration */ rc = globus_gram_job_manager_config_init(&config, argc, argv); if (rc != GLOBUS_SUCCESS) { reply_and_exit(NULL, rc, NULL); } globus_thread_key_create( &globus_i_gram_request_key, NULL); rc = globus_gram_job_manager_logging_init(&config); if (rc != GLOBUS_SUCCESS) { exit(1); } if (getenv("GRID_SECURITY_HTTP_BODY_FD") == NULL && !cgi_invoked) { debug_mode_service = GLOBUS_TRUE; } /* Set environment variables from configuration */ if(config.globus_location != NULL) { globus_libc_setenv("GLOBUS_LOCATION", config.globus_location, GLOBUS_TRUE); } if(config.tcp_port_range != NULL) { globus_libc_setenv("GLOBUS_TCP_PORT_RANGE", config.tcp_port_range, GLOBUS_TRUE); } if(config.tcp_source_range != NULL) { globus_libc_setenv("GLOBUS_TCP_SOURCE_RANGE", config.tcp_source_range, GLOBUS_TRUE); } /* Activate all of the modules we will be using */ rc = globus_l_gram_job_manager_activate(); if(rc != GLOBUS_SUCCESS) { exit(1); } /* * Get the delegated credential (or the default credential if we are * run without a client. Don't care about errors in the latter case. */ major_status = globus_gss_assist_acquire_cred( &minor_status, GSS_C_BOTH, &cred); if ((!debug_mode_service) && GSS_ERROR(major_status)) { globus_gss_assist_display_status( stderr, "Error acquiring security credential\n", major_status, minor_status, 0); exit(1); } if (cred != GSS_C_NO_CREDENTIAL) { unsigned long hash; char * newtag; rc = globus_gram_gsi_get_dn_hash( cred, &hash); if (rc == GLOBUS_SUCCESS) { newtag = globus_common_create_string("%s%s%lx", strcmp(config.service_tag, "untagged") == 0 ? "" : config.service_tag, strcmp(config.service_tag, "untagged") == 0 ? "" : ".", hash); free(config.service_tag); config.service_tag = newtag; } } /* * Remove delegated proxy from disk. */ if ((!debug_mode_service) && getenv("X509_USER_PROXY") != NULL) { remove(getenv("X509_USER_PROXY")); unsetenv("X509_USER_PROXY"); } /* Set up LRM-specific state based on our configuration. This will create * the job contact listener, start the SEG if needed, and open the log * file if needed. */ rc = globus_gram_job_manager_init(&manager, cred, &config); if(rc != GLOBUS_SUCCESS) { reply_and_exit(NULL, rc, manager.gt3_failure_message); } /* * Pull out file descriptor numbers for security context and job request * from the environment (set by the gatekeeper) */ if (cgi_invoked) { http_body_fd = 0; context_fd = -1; } else if (!debug_mode_service) { char * fd_env = getenv("GRID_SECURITY_HTTP_BODY_FD"); rc = sscanf(fd_env ? fd_env : "-1", "%d", &http_body_fd); if (rc != 1 || http_body_fd < 0) { fprintf(stderr, "Error locating http body fd\n"); exit(1); } fcntl(http_body_fd, F_SETFD, 1); fd_env = getenv("GRID_SECURITY_CONTEXT_FD"); rc = sscanf(fd_env ? fd_env : "-1", "%d", &context_fd); if (rc != 1 || context_fd < 0) { fprintf(stderr, "Error locating security context fd\n"); exit(1); } fcntl(context_fd, F_SETFD, 1); } /* Redirect stdin from /dev/null, we'll handle stdout after the reply is * sent */ if (!cgi_invoked) { freopen("/dev/null", "r", stdin); } /* Here we'll either become the active job manager to process all * jobs for this user/host/lrm combination, or we'll hand off the * file descriptors containing the info to the active job manager */ while (!located_active_jm) { /* We'll try to get the lock file associated with being the * active job manager here. If we get the OLD_JM_ALIVE error * somebody else has it */ rc = globus_gram_job_manager_startup_lock( &manager, &manager.lock_fd); if (rc == GLOBUS_SUCCESS) { /* We've acquired the lock. We will fork a new process to act like * all other job managers which don't have the lock, and continue * on in this process managing jobs for this LRM. Note that the * child process does not inherit the lock */ if (!debug_mode_service) { int save_errno = 0; /* We've acquired the manager lock */ forked_starter = fork(); save_errno = errno; if (forked_starter < 0) { if (sleeptime != 0) { sleep(sleeptime); } fprintf(stderr, "fork failed: %s", strerror(save_errno)); exit(1); } else if (forked_starter == 0) { /* We are the child process. We'll close our reference to * the lock and let the other process deal with jobs */ close(manager.lock_fd); manager.lock_fd = -1; } globus_logging_update_pid(); if (sleeptime != 0) { sleep(sleeptime); } } if (manager.lock_fd >= 0) { /* We hold the manager lock, so we'll store our credential, and * then, try to accept socket connections. If the socket * connections fail, we'll exit, and another process * will be forked to handle them. */ rc = globus_gram_job_manager_gsi_write_credential( NULL, cred, manager.cred_path); if (rc != GLOBUS_SUCCESS) { fprintf(stderr, "write cred failed\n"); exit(1); } if (!debug_mode_service) { close(http_body_fd); http_body_fd = -1; } rc = globus_gram_job_manager_startup_socket_init( &manager, &manager.active_job_manager_handle, &manager.socket_fd); if (rc != GLOBUS_SUCCESS) { /* This releases our lock. Either the child process will * attempt to acquire the lock again or some another job * manager will acquire the lock */ exit(0); } assert(manager.socket_fd != -1); } } else if (rc != GLOBUS_GRAM_PROTOCOL_ERROR_OLD_JM_ALIVE) { /* Some system error. Try again */ if (--lock_tries_left == 0) { reply_and_exit(NULL, rc, "Unable to create lock file"); } sleep(1); continue; } /* If manager.socket_fd != -1 then we are the main job manager for this * LRM. * We will restart all existing jobs and then allow the startup * socket to accept new jobs from other job managers. */ if (manager.socket_fd != -1) { /* Look up cputype/manufacturer if not known yet */ globus_l_gram_cputype_and_manufacturer(manager.config); GlobusTimeAbstimeGetCurrent(manager.usagetracker->jm_start_time); globus_i_gram_usage_stats_init(&manager); globus_i_gram_usage_start_session_stats(&manager); located_active_jm = GLOBUS_TRUE; /* Load existing jobs. The show must go on if this fails, unless it * fails with a misconfiguration error */ rc = globus_gram_job_manager_request_load_all( &manager); if (rc == GLOBUS_GRAM_PROTOCOL_ERROR_GATEKEEPER_MISCONFIGURED) { if (forked_starter > 0) { kill(forked_starter, SIGTERM); forked_starter = 0; } reply_and_exit(NULL, rc, manager.gt3_failure_message); } if (context_fd != -1) { close(context_fd); context_fd = -1; } freopen("/dev/null", "a", stdout); /* At this point, seg_last_timestamp is the earliest last timestamp * for any pre-existing jobs. If that is 0, then we don't have any * existing jobs so we'll just ignore seg events prior to now. */ if (manager.seg_last_timestamp == 0) { manager.seg_last_timestamp = time(NULL); } /* Start off the SEG if we need it. */ if (config.seg_module != NULL || strcmp(config.jobmanager_type, "fork") == 0 || strcmp(config.jobmanager_type, "condor") == 0) { rc = globus_gram_job_manager_init_seg(&manager); /* TODO: If SEG load fails and load_all added some to the * job_id hash, they will need to be pushed into the state * machine so that polling fallback can happen. */ if (rc != GLOBUS_SUCCESS) { config.seg_module = NULL; } } /* GRAM-128: * Register a periodic event to process the GRAM jobs that were * reloaded from their job state files at job manager start time. * This will acquire and then release a reference to each job, * which, behind the scenes, will kick of the state machine * for that job if needed. */ if (!globus_list_empty(manager.pending_restarts)) { globus_reltime_t restart_period; GlobusTimeReltimeSet(restart_period, 1, 0); rc = globus_callback_register_periodic( &manager.pending_restart_handle, NULL, &restart_period, globus_l_gram_process_pending_restarts, &manager); } { globus_reltime_t expire_period; GlobusTimeReltimeSet(expire_period, 1, 0); rc = globus_callback_register_periodic( &manager.expiration_handle, NULL, &expire_period, globus_gram_job_manager_expire_old_jobs, &manager); } { globus_reltime_t lockcheck_period; GlobusTimeReltimeSet(lockcheck_period, 60, 0); rc = globus_callback_register_periodic( &manager.lockcheck_handle, NULL, &lockcheck_period, globus_l_gram_lockcheck, &manager); } { globus_reltime_t idlescript_period; GlobusTimeReltimeSet(idlescript_period, 60, 0); rc = globus_callback_register_periodic( &manager.idle_script_handle, NULL, &idlescript_period, globus_gram_script_close_idle, &manager); } } else if (http_body_fd >= 0) { /* If manager.socket_fd == -1 then we are either the child from the * fork or another process started somehow (either command-line * invocation or via a job submit). If we have a client, then we'll * send our fds to the job manager with the lock and let it process * the job. * * If this succeeds, we set located_active_jm and leave the loop. * Otherwise, we try again. */ if (context_fd >= 0) { rc = globus_gram_job_manager_starter_send( &manager, http_body_fd, context_fd, fileno(stdout), cred); } else { rc = globus_gram_job_manager_starter_send_v2( &manager, cred); } if (rc == GLOBUS_SUCCESS) { located_active_jm = GLOBUS_TRUE; close(http_body_fd); if (context_fd >= 0) { close(context_fd); } manager.done = GLOBUS_TRUE; } else { globus_libc_usleep(250000); } } else { /* We were started by hand, but another process is currently the * main job manager */ unsigned long realpid = 0; FILE * pidin = fopen(manager.pid_path, "r"); fscanf(pidin, "%lu", &realpid); fclose(pidin); fprintf(stderr, "Other job manager process with pid %lu running and processing jobs\n", realpid); exit(0); } } /* Ignore SIGCHILD, and automatically reap child processes. Because of the * fork() above to delegate to another job manager process, and the use of * sub-processes to invoke the perl modules, we create some other * processes. We don't care too much how they exit, so we'll just make sure * we don't create zombies out of them. */ { struct sigaction act; act.sa_handler = SIG_IGN; sigemptyset(&act.sa_mask); sigaddset(&act.sa_mask, SIGCHLD); #ifdef SA_NOCLDWAIT act.sa_flags = SA_NOCLDWAIT; #else /* This may leave zombies running on non-POSIX systems like Hurd */ act.sa_flags = 0; #endif sigaction(SIGCHLD, &act, NULL); } /* Enable log rotation via SIGUSR1 */ { struct sigaction act; act.sa_handler = globus_i_job_manager_log_rotate; sigemptyset(&act.sa_mask); sigaddset(&act.sa_mask, SIGUSR1); act.sa_flags = 0; sigaction(SIGUSR1, &act, NULL); } GlobusGramJobManagerLock(&manager); if (manager.socket_fd != -1 && globus_hashtable_empty(&manager.request_hash) && manager.grace_period_timer == GLOBUS_NULL_HANDLE) { globus_gram_job_manager_set_grace_period_timer(&manager); } /* For the active job manager, this will block until all jobs have * terminated. For any other job manager, the monitor.done is set to * GLOBUS_TRUE and this falls right through. */ while (! manager.done) { GlobusGramJobManagerWait(&manager); } if (manager.expiration_handle != GLOBUS_NULL_HANDLE) { globus_callback_unregister(manager.expiration_handle, NULL, NULL, NULL); } if (manager.lockcheck_handle != GLOBUS_NULL_HANDLE) { globus_callback_unregister(manager.lockcheck_handle, NULL, NULL, NULL); } if (manager.idle_script_handle != GLOBUS_NULL_HANDLE) { globus_callback_unregister(manager.idle_script_handle, NULL, NULL, NULL); } GlobusGramJobManagerUnlock(&manager); globus_gram_job_manager_log( &manager, GLOBUS_GRAM_JOB_MANAGER_LOG_DEBUG, "event=gram.end " "level=DEBUG " "\n"); /* Clean-up to do if we are the active job manager only */ if (manager.socket_fd != -1) { globus_gram_job_manager_script_close_all(&manager); globus_i_gram_usage_end_session_stats(&manager); globus_i_gram_usage_stats_destroy(&manager); remove(manager.pid_path); remove(manager.cred_path); remove(manager.socket_path); remove(manager.lock_path); } globus_gram_job_manager_logging_destroy(); globus_gram_job_manager_destroy(&manager); globus_gram_job_manager_config_destroy(&config); rc = globus_l_gram_deactivate(); if (rc != GLOBUS_SUCCESS) { fprintf(stderr, "deactivation failed with rc=%d\n", rc); exit(1); } /* { const char * gk_jm_id_var = "GATEKEEPER_JM_ID"; const char * gk_jm_id = globus_libc_getenv(gk_jm_id_var); globus_gram_job_manager_request_acct( request, "%s %s JM exiting\n", gk_jm_id_var, gk_jm_id ? gk_jm_id : "none"); } */ return(0); }
/** * load config file. substitute $BUILD_FLAVOR for build_flavor, */ static int globus_l_extension_activate(void) { static globus_bool_t initialized = GLOBUS_FALSE; globus_result_t result = GLOBUS_SUCCESS; GlobusFuncName(globus_l_extension_activate); if(!initialized) { GlobusDebugInit(GLOBUS_EXTENSION, TRACE VERBOSE DLL); GlobusExtensionDebugEnter(); globus_rmutex_init(&globus_l_libtool_mutex, NULL); globus_thread_key_create(&globus_l_libtool_key, NULL); # ifndef BUILD_STATIC_ONLY if(lt_dlinit() != 0) { goto error_dlinit; } # if HAVE_LT_DLMUTEX_REGISTER if(lt_dlmutex_register( globus_l_libtool_mutex_lock, globus_l_libtool_mutex_unlock, globus_l_libtool_set_error, globus_l_libtool_get_error) != 0) { goto error_dlmutex; } # endif # endif /* !BUILD_STATIC_ONLY */ globus_hashtable_init( &globus_l_extension_loaded, 32, globus_hashtable_string_hash, globus_hashtable_string_keyeq); globus_hashtable_init( &globus_l_extension_builtins, 32, globus_hashtable_string_hash, globus_hashtable_string_keyeq); globus_rmutex_init(&globus_l_extension_mutex, NULL); globus_thread_key_create(&globus_l_extension_owner_key, NULL); result = globus_eval_path("${libdir}", &globus_l_globus_location); if(result != GLOBUS_SUCCESS) { goto error_path; } initialized = GLOBUS_TRUE; GlobusExtensionDebugExit(); } return GLOBUS_SUCCESS; error_path: globus_hashtable_destroy(&globus_l_extension_builtins); globus_hashtable_destroy(&globus_l_extension_loaded); # ifndef BUILD_STATIC_ONLY # if HAVE_LT_DLMUTEX_REGISTER error_dlmutex: # endif lt_dlexit(); error_dlinit: GlobusExtensionDebugExitWithError(); return GLOBUS_FAILURE; # endif /* !BUILD_STATIC_ONLY */ }