/** * Parses the request body, logs any REQUEST messages to the activity log, * stamps the translator ingress on each message, and returns the updated * messages as a JSON string. */ static char* osrfHttpTranslatorParseRequest(osrfHttpTranslator* trans) { osrfMessage* msg; osrfMessage* msgList[MAX_MSGS_PER_PACKET]; int numMsgs = osrf_message_deserialize(trans->body, msgList, MAX_MSGS_PER_PACKET); osrfLogDebug(OSRF_LOG_MARK, "parsed %d opensrf messages in this packet", numMsgs); if(numMsgs == 0) return NULL; // log request messages to the activity log int i; for(i = 0; i < numMsgs; i++) { msg = msgList[i]; osrfMessageSetIngress(msg, TRANSLATOR_INGRESS); switch(msg->m_type) { case REQUEST: { const jsonObject* params = msg->_params; growing_buffer* act = buffer_init(128); char* method = msg->method_name; buffer_fadd(act, "[%s] [%s] %s %s", trans->remoteHost, "", trans->service, method); const jsonObject* obj = NULL; int i = 0; const char* str; int redactParams = 0; while( (str = osrfStringArrayGetString(log_protect_arr, i++)) ) { //osrfLogInternal(OSRF_LOG_MARK, "Checking for log protection [%s]", str); if(!strncmp(method, str, strlen(str))) { redactParams = 1; break; } } if(redactParams) { OSRF_BUFFER_ADD(act, " **PARAMS REDACTED**"); } else { i = 0; while((obj = jsonObjectGetIndex(params, i++))) { str = jsonObjectToJSON(obj); if( i == 1 ) OSRF_BUFFER_ADD(act, " "); else OSRF_BUFFER_ADD(act, ", "); OSRF_BUFFER_ADD(act, str); free(str); } } osrfLogActivity(OSRF_LOG_MARK, "%s", act->buf); buffer_free(act); break; } case CONNECT: trans->connecting = 1; if (numMsgs == 1) trans->connectOnly = 1; break; case DISCONNECT: trans->disconnecting = 1; if (numMsgs == 1) trans->disconnectOnly = 1; break; case RESULT: osrfLogWarning( OSRF_LOG_MARK, "Unexpected RESULT message received" ); break; case STATUS: osrfLogWarning( OSRF_LOG_MARK, "Unexpected STATUS message received" ); break; default: osrfLogWarning( OSRF_LOG_MARK, "Invalid message type %d received", msg->m_type ); break; } } char* jsonString = osrfMessageSerializeBatch(msgList, numMsgs); for(i = 0; i < numMsgs; i++) { osrfMessageFree(msgList[i]); } return jsonString; }
/** @brief Initialize the application. @return Zero if successful, or non-zero if not. Load the IDL file into an internal data structure for future reference. Each non-virtual class in the IDL corresponds to a table or view in the database, or to a subquery defined in the IDL. Ignore all virtual tables and virtual fields. Register a number of methods, some of them general-purpose and others specific for particular classes. The name of the application is given by the MODULENAME macro, whose value depends on conditional compilation. The method names also incorporate MODULENAME, followed by a dot, as a prefix. The general-purpose methods are as follows (minus their MODULENAME prefixes): - json_query - transaction.begin - transaction.commit - transaction.rollback - savepoint.set - savepoint.release - savepoint.rollback For each non-virtual class, create up to eight class-specific methods: - create (not for readonly classes) - retrieve - update (not for readonly classes) - delete (not for readonly classes - search (atomic and non-atomic versions) - id_list (atomic and non-atomic versions) The full method names follow the pattern "MODULENAME.direct.XXX.method_type", where XXX is the fieldmapper name from the IDL, with every run of one or more consecutive colons replaced by a period. In addition, the names of atomic methods have a suffix of ".atomic". This function is called when the registering the application, and is executed by the listener before spawning the drones. */ int osrfAppInitialize( void ) { osrfLogInfo(OSRF_LOG_MARK, "Initializing the CStore Server..."); osrfLogInfo(OSRF_LOG_MARK, "Finding XML file..."); // Load the IDL into memory if ( !oilsIDLInit( osrf_settings_host_value( "/IDL" ))) return 1; /* return non-zero to indicate error */ // Open the database temporarily. Look up the datatypes of all // the non-virtual fields and record them with the IDL data. dbi_conn handle = oilsConnectDB( modulename ); if( !handle ) return -1; else if( oilsExtendIDL( handle )) { osrfLogError( OSRF_LOG_MARK, "Error extending the IDL" ); return -1; } dbi_conn_close( handle ); // Get the maximum flesh depth from the settings char* md = osrf_settings_host_value( "/apps/%s/app_settings/max_query_recursion", modulename ); int max_flesh_depth = 100; if( md ) max_flesh_depth = atoi( md ); if( max_flesh_depth < 0 ) max_flesh_depth = 1; else if( max_flesh_depth > 1000 ) max_flesh_depth = 1000; oilsSetSQLOptions( modulename, enforce_pcrud, max_flesh_depth ); // Now register all the methods growing_buffer* method_name = buffer_init(64); // Generic search thingy buffer_add( method_name, modulename ); buffer_add( method_name, ".json_query" ); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR( method_name ), "doJSONSearch", "", 1, OSRF_METHOD_STREAMING ); // Next we register all the transaction and savepoint methods buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".transaction.begin"); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR( method_name ), "beginTransaction", "", 0, 0 ); buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".transaction.commit"); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR(method_name), "commitTransaction", "", 0, 0 ); buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".transaction.rollback"); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR(method_name), "rollbackTransaction", "", 0, 0 ); buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".savepoint.set"); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR(method_name), "setSavepoint", "", 1, 0 ); buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".savepoint.release"); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR(method_name), "releaseSavepoint", "", 1, 0 ); buffer_reset(method_name); OSRF_BUFFER_ADD(method_name, modulename ); OSRF_BUFFER_ADD(method_name, ".savepoint.rollback"); osrfAppRegisterMethod( modulename, OSRF_BUFFER_C_STR(method_name), "rollbackSavepoint", "", 1, 0 ); static const char* global_method[] = { "create", "retrieve", "update", "delete", "search", "id_list" }; const int global_method_count = sizeof( global_method ) / sizeof ( global_method[0] ); unsigned long class_count = osrfHashGetCount( oilsIDL() ); osrfLogDebug(OSRF_LOG_MARK, "%lu classes loaded", class_count ); osrfLogDebug(OSRF_LOG_MARK, "At most %lu methods will be generated", (unsigned long) (class_count * global_method_count) ); osrfHashIterator* class_itr = osrfNewHashIterator( oilsIDL() ); osrfHash* idlClass = NULL; // For each class in the IDL... while( (idlClass = osrfHashIteratorNext( class_itr ) ) ) { const char* classname = osrfHashIteratorKey( class_itr ); osrfLogInfo(OSRF_LOG_MARK, "Generating class methods for %s", classname); if (!osrfStringArrayContains( osrfHashGet(idlClass, "controller"), modulename )) { osrfLogInfo(OSRF_LOG_MARK, "%s is not listed as a controller for %s, moving on", modulename, classname); continue; } if ( str_is_true( osrfHashGet(idlClass, "virtual") ) ) { osrfLogDebug(OSRF_LOG_MARK, "Class %s is virtual, skipping", classname ); continue; } // Look up some other attributes of the current class const char* idlClass_fieldmapper = osrfHashGet(idlClass, "fieldmapper"); if( !idlClass_fieldmapper ) { osrfLogDebug( OSRF_LOG_MARK, "Skipping class \"%s\"; no fieldmapper in IDL", classname ); continue; } const char* readonly = osrfHashGet(idlClass, "readonly"); int i; for( i = 0; i < global_method_count; ++i ) { // for each global method const char* method_type = global_method[ i ]; osrfLogDebug(OSRF_LOG_MARK, "Using files to build %s class methods for %s", method_type, classname); // No create, update, or delete methods for a readonly class if ( str_is_true( readonly ) && ( *method_type == 'c' || *method_type == 'u' || *method_type == 'd') ) continue; buffer_reset( method_name ); // Build the method name: MODULENAME.MODULENAME.direct.XXX.method_type // where XXX is the fieldmapper name from the IDL, with every run of // one or more consecutive colons replaced by a period. char* st_tmp = NULL; char* part = NULL; char* _fm = strdup( idlClass_fieldmapper ); part = strtok_r(_fm, ":", &st_tmp); buffer_fadd(method_name, "%s.direct.%s", modulename, part); while ((part = strtok_r(NULL, ":", &st_tmp))) { OSRF_BUFFER_ADD_CHAR(method_name, '.'); OSRF_BUFFER_ADD(method_name, part); } OSRF_BUFFER_ADD_CHAR(method_name, '.'); OSRF_BUFFER_ADD(method_name, method_type); free(_fm); // For an id_list or search method we specify the OSRF_METHOD_STREAMING option. // The consequence is that we implicitly create an atomic method in addition to // the usual non-atomic method. int flags = 0; if (*method_type == 'i' || *method_type == 's') { // id_list or search flags = flags | OSRF_METHOD_STREAMING; } osrfHash* method_meta = osrfNewHash(); osrfHashSet( method_meta, idlClass, "class"); osrfHashSet( method_meta, buffer_data( method_name ), "methodname" ); osrfHashSet( method_meta, strdup(method_type), "methodtype" ); // Register the method, with a pointer to an osrfHash to tell the method // its name, type, and class. osrfAppRegisterExtendedMethod( modulename, OSRF_BUFFER_C_STR( method_name ), "dispatchCRUDMethod", "", 1, flags, (void*)method_meta ); } // end for each global method } // end for each class in IDL buffer_free( method_name ); osrfHashIteratorFree( class_itr ); return 0; }
static char* extract_inbound_messages( const request_rec *r, const char* service, const char* thread, const char* recipient, const jsonObject *osrf_msg) { int i; int num_msgs = osrf_msg->size; osrfMessage* msg; osrfMessage* msg_list[num_msgs]; // here we do an extra json round-trip to get the data // in a form osrf_message_deserialize can understand // TODO: consider a version of osrf_message_init which can // accept a jsonObject* instead of a JSON string. char *osrf_msg_json = jsonObjectToJSON(osrf_msg); osrf_message_deserialize(osrf_msg_json, msg_list, num_msgs); free(osrf_msg_json); // should we require the caller to always pass the service? if (service == NULL) service = ""; for(i = 0; i < num_msgs; i++) { msg = msg_list[i]; osrfMessageSetIngress(msg, WEBSOCKET_TRANSLATOR_INGRESS); switch(msg->m_type) { case REQUEST: { const jsonObject* params = msg->_params; growing_buffer* act = buffer_init(128); char* method = msg->method_name; buffer_fadd(act, "[%s] [%s] %s %s", get_client_ip(r), "", service, method); const jsonObject* obj = NULL; int i = 0; const char* str; int redactParams = 0; while( (str = osrfStringArrayGetString(log_protect_arr, i++)) ) { if(!strncmp(method, str, strlen(str))) { redactParams = 1; break; } } if(redactParams) { OSRF_BUFFER_ADD(act, " **PARAMS REDACTED**"); } else { i = 0; while((obj = jsonObjectGetIndex(params, i++))) { char* str = jsonObjectToJSON(obj); if( i == 1 ) OSRF_BUFFER_ADD(act, " "); else OSRF_BUFFER_ADD(act, ", "); OSRF_BUFFER_ADD(act, str); free(str); } } osrfLogActivity(OSRF_LOG_MARK, "%s", act->buf); buffer_free(act); requests_in_flight++; break; } case DISCONNECT: clear_cached_recipient(thread); break; } } char* finalMsg = osrfMessageSerializeBatch(msg_list, num_msgs); // clean up our messages for(i = 0; i < num_msgs; i++) osrfMessageFree(msg_list[i]); return finalMsg; }
static int osrf_json_gateway_method_handler (request_rec *r) { /* make sure we're needed first thing*/ if (strcmp(r->handler, MODULE_NAME )) return DECLINED; osrf_json_gateway_dir_config* dir_conf = ap_get_module_config(r->per_dir_config, &osrf_json_gateway_module); /* provide 2 different JSON parsers and serializers to support legacy JSON */ jsonObject* (*parseJSONFunc) (const char*) = legacy_jsonParseString; char* (*jsonToStringFunc) (const jsonObject*) = legacy_jsonObjectToJSON; if(dir_conf->legacyJSON) { ap_log_rerror( APLOG_MARK, APLOG_DEBUG, 0, r, "Using legacy JSON"); } else { parseJSONFunc = jsonParse; jsonToStringFunc = jsonObjectToJSON; } osrfLogDebug(OSRF_LOG_MARK, "osrf gateway: entered request handler"); /* verify we are connected */ if( !bootstrapped || !osrfSystemGetTransportClient()) { ap_log_rerror( APLOG_MARK, APLOG_ERR, 0, r, "Cannot process request " "because the OpenSRF JSON gateway has not been bootstrapped..."); usleep( 100000 ); /* 100 milliseconds */ exit(1); } osrfLogSetAppname("osrf_json_gw"); char* osrf_locale = NULL; char* param_locale = NULL; /* locale for this call */ char* service = NULL; /* service to connect to */ char* method = NULL; /* method to perform */ char* format = NULL; /* method to perform */ char* a_l = NULL; /* request api level */ char* input_format = NULL; /* POST data format, defaults to 'format' */ int isXML = 0; int api_level = 1; r->allowed |= (AP_METHOD_BIT << M_GET); r->allowed |= (AP_METHOD_BIT << M_POST); osrfLogDebug(OSRF_LOG_MARK, "osrf gateway: parsing URL params"); osrfStringArray* mparams = NULL; osrfStringArray* params = apacheParseParms(r); /* free me */ param_locale = apacheGetFirstParamValue( params, "locale" ); service = apacheGetFirstParamValue( params, "service" ); method = apacheGetFirstParamValue( params, "method" ); format = apacheGetFirstParamValue( params, "format" ); input_format = apacheGetFirstParamValue( params, "input_format" ); a_l = apacheGetFirstParamValue( params, "api_level" ); mparams = apacheGetParamValues( params, "param" ); /* free me */ if(format == NULL) format = strdup( "json" ); if(input_format == NULL) input_format = strdup( format ); /* set the user defined timeout value */ int timeout = 60; char* tout = apacheGetFirstParamValue( params, "timeout" ); /* request timeout in seconds */ if( tout ) { timeout = atoi(tout); osrfLogDebug(OSRF_LOG_MARK, "Client supplied timeout of %d", timeout); free( tout ); } if (a_l) { api_level = atoi(a_l); free( a_l ); } if (!strcasecmp(format, "xml")) { isXML = 1; ap_set_content_type(r, "application/xml"); } else { ap_set_content_type(r, "text/plain"); } free( format ); int ret = OK; /* ----------------------------------------------------------------- */ /* Grab the requested locale using the Accept-Language header*/ if ( !param_locale ) { if ( apr_table_get(r->headers_in, "X-OpenSRF-Language") ) { param_locale = strdup( apr_table_get(r->headers_in, "X-OpenSRF-Language") ); } else if ( apr_table_get(r->headers_in, "Accept-Language") ) { param_locale = strdup( apr_table_get(r->headers_in, "Accept-Language") ); } } if (param_locale) { growing_buffer* osrf_locale_buf = buffer_init(16); if (index(param_locale, ',')) { int ind = index(param_locale, ',') - param_locale; int i; for ( i = 0; i < ind && i < 128; i++ ) buffer_add_char( osrf_locale_buf, param_locale[i] ); } else { buffer_add( osrf_locale_buf, param_locale ); } free(param_locale); osrf_locale = buffer_release( osrf_locale_buf ); } else { osrf_locale = strdup( osrf_json_default_locale ); } /* ----------------------------------------------------------------- */ if(!(service && method)) { osrfLogError(OSRF_LOG_MARK, "Service [%s] not found or not allowed", service); ret = HTTP_NOT_FOUND; } else { /* This will log all heaers to the apache error log const apr_array_header_t* arr = apr_table_elts(r->headers_in); const void* ptr; while( (ptr = apr_array_pop(arr)) ) { apr_table_entry_t* e = (apr_table_entry_t*) ptr; fprintf(stderr, "Table entry: %s : %s\n", e->key, e->val ); } fflush(stderr); */ osrfAppSession* session = osrfAppSessionClientInit(service); osrf_app_session_set_locale(session, osrf_locale); double starttime = get_timestamp_millis(); int req_id = -1; if(!strcasecmp(input_format, "json")) { jsonObject * arr = jsonNewObject(NULL); const char* str; int i = 0; while( (str = osrfStringArrayGetString(mparams, i++)) ) jsonObjectPush(arr, parseJSONFunc(str)); req_id = osrfAppSessionSendRequest( session, arr, method, api_level ); jsonObjectFree(arr); } else { /** * If we receive XML method params, convert each param to a JSON object * and pass the array of JSON object params to the method */ if(!strcasecmp(input_format, "xml")) { jsonObject* jsonParams = jsonNewObject(NULL); const char* str; int i = 0; while( (str = osrfStringArrayGetString(mparams, i++)) ) { jsonObjectPush(jsonParams, jsonXMLToJSONObject(str)); } req_id = osrfAppSessionSendRequest( session, jsonParams, method, api_level ); jsonObjectFree(jsonParams); } } if( req_id == -1 ) { osrfLogError(OSRF_LOG_MARK, "I am unable to communicate with opensrf..going away..."); osrfAppSessionFree(session); /* we don't want to spawn an intense re-forking storm * if there is no jabber server.. so give it some time before we die */ usleep( 100000 ); /* 100 milliseconds */ exit(1); } /* ----------------------------------------------------------------- */ /* log all requests to the activity log */ const char* authtoken = apr_table_get(r->headers_in, "X-OILS-Authtoken"); if(!authtoken) authtoken = ""; growing_buffer* act = buffer_init(128); buffer_fadd(act, "[%s] [%s] [%s] %s %s", r->connection->remote_ip, authtoken, osrf_locale, service, method ); const char* str; int i = 0; while( (str = osrfStringArrayGetString(mparams, i++)) ) { if( i == 1 ) { OSRF_BUFFER_ADD(act, " "); OSRF_BUFFER_ADD(act, str); } else { OSRF_BUFFER_ADD(act, ", "); OSRF_BUFFER_ADD(act, str); } } osrfLogActivity( OSRF_LOG_MARK, act->buf ); buffer_free(act); /* ----------------------------------------------------------------- */ osrfMessage* omsg = NULL; int statuscode = 200; /* kick off the object */ if (isXML) ap_rputs( "<response xmlns=\"http://opensrf.org/-/namespaces/gateway/v1\"><payload>", r ); else ap_rputs("{\"payload\":[", r); int morethan1 = 0; char* statusname = NULL; char* statustext = NULL; char* output = NULL; while((omsg = osrfAppSessionRequestRecv( session, req_id, timeout ))) { statuscode = omsg->status_code; const jsonObject* res; if( ( res = osrfMessageGetResult(omsg)) ) { if (isXML) { output = jsonObjectToXML( res ); } else { output = jsonToStringFunc( res ); if( morethan1 ) ap_rputs(",", r); /* comma between JSON array items */ } ap_rputs(output, r); free(output); morethan1 = 1; } else { if( statuscode > 299 ) { /* the request returned a low level error */ statusname = omsg->status_name ? strdup(omsg->status_name) : strdup("Unknown Error"); statustext = omsg->status_text ? strdup(omsg->status_text) : strdup("No Error Message"); osrfLogError( OSRF_LOG_MARK, "Gateway received error: %s", statustext ); } } osrfMessageFree(omsg); if(statusname) break; } double duration = get_timestamp_millis() - starttime; osrfLogDebug(OSRF_LOG_MARK, "gateway request took %f seconds", duration); if (isXML) ap_rputs("</payload>", r); else ap_rputs("]",r); /* finish off the payload array */ if(statusname) { /* add a debug field if the request died */ ap_log_rerror( APLOG_MARK, APLOG_INFO, 0, r, "OpenSRF JSON Request returned error: %s -> %s", statusname, statustext ); int l = strlen(statusname) + strlen(statustext) + 32; char buf[l]; if (isXML) snprintf( buf, sizeof(buf), "<debug>\"%s : %s\"</debug>", statusname, statustext ); else { char bb[l]; snprintf(bb, sizeof(bb), "%s : %s", statusname, statustext); jsonObject* tmp = jsonNewObject(bb); char* j = jsonToStringFunc(tmp); snprintf( buf, sizeof(buf), ",\"debug\": %s", j); free(j); jsonObjectFree(tmp); } ap_rputs(buf, r); free(statusname); free(statustext); } /* insert the status code */ char buf[32]; if (isXML) snprintf(buf, sizeof(buf), "<status>%d</status>", statuscode ); else snprintf(buf, sizeof(buf), ",\"status\":%d", statuscode ); ap_rputs( buf, r ); if (isXML) ap_rputs("</response>", r); else ap_rputs( "}", r ); /* finish off the object */ osrfAppSessionFree(session); } osrfLogInfo(OSRF_LOG_MARK, "Completed processing service=%s, method=%s", service, method); osrfStringArrayFree(params); osrfStringArrayFree(mparams); free( osrf_locale ); free( input_format ); free( method ); free( service ); osrfLogDebug(OSRF_LOG_MARK, "Gateway served %d requests", ++numserved); osrfLogClearXid(); return ret; }
int _recurse_jsonObjectToXML(const jsonObject* obj, growing_buffer* res_xml) { char * hint = NULL; if (obj->classname) hint = strdup(obj->classname); if(obj->type == JSON_NULL) { if (hint) buffer_fadd(res_xml, "<null class_hint=\"%s\"/>",hint); else buffer_add(res_xml, "<null/>"); } else if(obj->type == JSON_BOOL) { const char* bool_val; if (obj->value.b) bool_val = "true"; else bool_val = "false"; if (hint) buffer_fadd(res_xml, "<boolean value=\"%s\" class_hint=\"%s\"/>", bool_val, hint); else buffer_fadd(res_xml, "<boolean value=\"%s\"/>", bool_val); } else if (obj->type == JSON_STRING) { if (hint) { char * t = _escape_xml(jsonObjectGetString(obj)); buffer_fadd(res_xml,"<string class_hint=\"%s\">%s</string>", hint, t); free(t); } else { char * t = _escape_xml(jsonObjectGetString(obj)); buffer_fadd(res_xml,"<string>%s</string>", t); free(t); } } else if(obj->type == JSON_NUMBER) { double x = jsonObjectGetNumber(obj); if (hint) { if (x == (int)x) buffer_fadd(res_xml,"<number class_hint=\"%s\">%d</number>", hint, (int)x); else buffer_fadd(res_xml,"<number class_hint=\"%s\">%lf</number>", hint, x); } else { if (x == (int)x) buffer_fadd(res_xml,"<number>%d</number>", (int)x); else buffer_fadd(res_xml,"<number>%lf</number>", x); } } else if (obj->type == JSON_ARRAY) { if (hint) buffer_fadd(res_xml,"<array class_hint=\"%s\">", hint); else buffer_add(res_xml,"<array>"); int i; for ( i = 0; i!= obj->size; i++ ) _recurse_jsonObjectToXML(jsonObjectGetIndex(obj,i), res_xml); buffer_add(res_xml,"</array>"); } else if (obj->type == JSON_HASH) { if (hint) buffer_fadd(res_xml,"<object class_hint=\"%s\">", hint); else buffer_add(res_xml,"<object>"); jsonIterator* itr = jsonNewIterator(obj); const jsonObject* tmp; while( (tmp = jsonIteratorNext(itr)) ) { buffer_fadd(res_xml,"<element key=\"%s\">",itr->key); _recurse_jsonObjectToXML(tmp, res_xml); buffer_add(res_xml,"</element>"); } jsonIteratorFree(itr); buffer_add(res_xml,"</object>"); } if (hint) free(hint); return 1; }