void catalog_request(JCR *jcr, BSOCK *bs) { MEDIA_DBR mr, sdmr; JOBMEDIA_DBR jm; char Job[MAX_NAME_LENGTH]; char pool_name[MAX_NAME_LENGTH]; int index, ok, label, writing; POOLMEM *omsg; POOL_DBR pr; uint32_t Stripe, Copy; uint64_t MediaId; utime_t VolFirstWritten; utime_t VolLastWritten; memset(&sdmr, 0, sizeof(sdmr)); memset(&jm, 0, sizeof(jm)); Dsm_check(100); /* * Request to find next appendable Volume for this Job */ Dmsg1(100, "catreq %s", bs->msg); if (!jcr->db) { omsg = get_memory(bs->msglen+1); pm_strcpy(omsg, bs->msg); bs->fsend(_("1990 Invalid Catalog Request: %s"), omsg); Jmsg1(jcr, M_FATAL, 0, _("Invalid Catalog request; DB not open: %s"), omsg); free_memory(omsg); return; } /* * Find next appendable medium for SD */ if (sscanf(bs->msg, Find_media, &Job, &index, &pool_name, &mr.MediaType) == 4) { memset(&pr, 0, sizeof(pr)); bstrncpy(pr.Name, pool_name, sizeof(pr.Name)); unbash_spaces(pr.Name); ok = db_get_pool_record(jcr, jcr->db, &pr); if (ok) { mr.PoolId = pr.PoolId; set_storageid_in_mr(jcr->res.wstore, &mr); mr.ScratchPoolId = pr.ScratchPoolId; ok = find_next_volume_for_append(jcr, &mr, index, fnv_create_vol, fnv_prune); Dmsg3(050, "find_media ok=%d idx=%d vol=%s\n", ok, index, mr.VolumeName); } /* * Send Find Media response to Storage daemon */ if (ok) { send_volume_info_to_storage_daemon(jcr, bs, &mr); } else { bs->fsend(_("1901 No Media.\n")); Dmsg0(500, "1901 No Media.\n"); } /* * Request to find specific Volume information */ } else if (sscanf(bs->msg, Get_Vol_Info, &Job, &mr.VolumeName, &writing) == 3) { Dmsg1(100, "CatReq GetVolInfo Vol=%s\n", mr.VolumeName); /* * Find the Volume */ unbash_spaces(mr.VolumeName); if (db_get_media_record(jcr, jcr->db, &mr)) { const char *reason = NULL; /* detailed reason for rejection */ /* * If we are reading, accept any volume (reason == NULL) * If we are writing, check if the Volume is valid * for this job, and do a recycle if necessary */ if (writing) { /* * SD wants to write this Volume, so make * sure it is suitable for this job, i.e. * Pool matches, and it is either Append or Recycle * and Media Type matches and Pool allows any volume. */ if (mr.PoolId != jcr->jr.PoolId) { reason = _("not in Pool"); } else if (!bstrcmp(mr.MediaType, jcr->res.wstore->media_type)) { reason = _("not correct MediaType"); } else { /* * Now try recycling if necessary * reason set non-NULL if we cannot use it */ check_if_volume_valid_or_recyclable(jcr, &mr, &reason); } } if (!reason && mr.Enabled != 1) { reason = _("is not Enabled"); } if (reason == NULL) { /* * Send Find Media response to Storage daemon */ send_volume_info_to_storage_daemon(jcr, bs, &mr); } else { /* Not suitable volume */ bs->fsend(_("1998 Volume \"%s\" catalog status is %s, %s.\n"), mr.VolumeName, mr.VolStatus, reason); } } else { bs->fsend(_("1997 Volume \"%s\" not in catalog.\n"), mr.VolumeName); Dmsg1(100, "1997 Volume \"%s\" not in catalog.\n", mr.VolumeName); } /* * Request to update Media record. Comes typically at the end * of a Storage daemon Job Session, when labeling/relabeling a * Volume, or when an EOF mark is written. */ } else if (sscanf(bs->msg, Update_media, &Job, &sdmr.VolumeName, &sdmr.VolJobs, &sdmr.VolFiles, &sdmr.VolBlocks, &sdmr.VolBytes, &sdmr.VolMounts, &sdmr.VolErrors, &sdmr.VolWrites, &sdmr.MaxVolBytes, &VolLastWritten, &sdmr.VolStatus, &sdmr.Slot, &label, &sdmr.InChanger, &sdmr.VolReadTime, &sdmr.VolWriteTime, &VolFirstWritten) == 18) { db_lock(jcr->db); Dmsg3(400, "Update media %s oldStat=%s newStat=%s\n", sdmr.VolumeName, mr.VolStatus, sdmr.VolStatus); bstrncpy(mr.VolumeName, sdmr.VolumeName, sizeof(mr.VolumeName)); /* copy Volume name */ unbash_spaces(mr.VolumeName); if (!db_get_media_record(jcr, jcr->db, &mr)) { Jmsg(jcr, M_ERROR, 0, _("Unable to get Media record for Volume %s: ERR=%s\n"), mr.VolumeName, db_strerror(jcr->db)); bs->fsend(_("1991 Catalog Request for vol=%s failed: %s"), mr.VolumeName, db_strerror(jcr->db)); goto bail_out; } /* Set first written time if this is first job */ if (mr.FirstWritten == 0) { if (VolFirstWritten == 0) { mr.FirstWritten = jcr->start_time; /* use Job start time as first write */ } else { mr.FirstWritten = VolFirstWritten; } mr.set_first_written = true; } /* If we just labeled the tape set time */ if (label || mr.LabelDate == 0) { mr.LabelDate = jcr->start_time; mr.set_label_date = true; if (mr.InitialWrite == 0) { mr.InitialWrite = jcr->start_time; } Dmsg2(400, "label=%d labeldate=%d\n", label, mr.LabelDate); } else { /* * Insanity check for VolFiles get set to a smaller value */ if (sdmr.VolFiles < mr.VolFiles) { Jmsg(jcr, M_FATAL, 0, _("Volume Files at %u being set to %u" " for Volume \"%s\". This is incorrect.\n"), mr.VolFiles, sdmr.VolFiles, mr.VolumeName); bs->fsend(_("1992 Update Media error. VolFiles=%u, CatFiles=%u\n"), sdmr.VolFiles, mr.VolFiles); goto bail_out; } } Dmsg2(400, "Update media: BefVolJobs=%u After=%u\n", mr.VolJobs, sdmr.VolJobs); /* * Check if the volume has been written by the job, * and update the LastWritten field if needed. */ if (mr.VolBlocks != sdmr.VolBlocks && VolLastWritten != 0) { mr.LastWritten = VolLastWritten; } /* * Update to point to the last device used to write the Volume. * However, do so only if we are writing the tape, i.e. * the number of VolWrites has increased. */ if (jcr->res.wstore && sdmr.VolWrites > mr.VolWrites) { Dmsg2(050, "Update StorageId old=%d new=%d\n", mr.StorageId, jcr->res.wstore->StorageId); /* Update StorageId after write */ set_storageid_in_mr(jcr->res.wstore, &mr); } else { /* Nothing written, reset same StorageId */ set_storageid_in_mr(NULL, &mr); } /* Copy updated values to original media record */ mr.VolJobs = sdmr.VolJobs; mr.VolFiles = sdmr.VolFiles; mr.VolBlocks = sdmr.VolBlocks; mr.VolBytes = sdmr.VolBytes; mr.VolMounts = sdmr.VolMounts; mr.VolErrors = sdmr.VolErrors; mr.VolWrites = sdmr.VolWrites; mr.Slot = sdmr.Slot; mr.InChanger = sdmr.InChanger; bstrncpy(mr.VolStatus, sdmr.VolStatus, sizeof(mr.VolStatus)); mr.VolReadTime = sdmr.VolReadTime; mr.VolWriteTime = sdmr.VolWriteTime; Dmsg2(400, "db_update_media_record. Stat=%s Vol=%s\n", mr.VolStatus, mr.VolumeName); /* * Update the database, then before sending the response to the * SD, check if the Volume has expired. */ if (!db_update_media_record(jcr, jcr->db, &mr)) { Jmsg(jcr, M_FATAL, 0, _("Catalog error updating Media record. %s"), db_strerror(jcr->db)); bs->fsend(_("1993 Update Media error\n")); Dmsg0(400, "send error\n"); } else { (void)has_volume_expired(jcr, &mr); send_volume_info_to_storage_daemon(jcr, bs, &mr); } bail_out: db_unlock(jcr->db); Dmsg1(400, ">CatReq response: %s", bs->msg); Dmsg1(400, "Leave catreq jcr 0x%x\n", jcr); return; /* * Request to create a JobMedia record */ } else if (sscanf(bs->msg, Create_job_media, &Job, &jm.FirstIndex, &jm.LastIndex, &jm.StartFile, &jm.EndFile, &jm.StartBlock, &jm.EndBlock, &Copy, &Stripe, &MediaId) == 10) { if (jcr->mig_jcr) { jm.JobId = jcr->mig_jcr->JobId; } else { jm.JobId = jcr->JobId; } jm.MediaId = MediaId; Dmsg6(400, "create_jobmedia JobId=%d MediaId=%d SF=%d EF=%d FI=%d LI=%d\n", jm.JobId, jm.MediaId, jm.StartFile, jm.EndFile, jm.FirstIndex, jm.LastIndex); if (!db_create_jobmedia_record(jcr, jcr->db, &jm)) { Jmsg(jcr, M_FATAL, 0, _("Catalog error creating JobMedia record. %s"), db_strerror(jcr->db)); bs->fsend(_("1992 Create JobMedia error\n")); } else { Dmsg0(400, "JobMedia record created\n"); bs->fsend(OK_create); } } else { omsg = get_memory(bs->msglen+1); pm_strcpy(omsg, bs->msg); bs->fsend(_("1990 Invalid Catalog Request: %s"), omsg); Jmsg1(jcr, M_FATAL, 0, _("Invalid Catalog request: %s"), omsg); free_memory(omsg); } Dmsg1(400, ">CatReq response: %s", bs->msg); Dmsg1(400, "Leave catreq jcr 0x%x\n", jcr); return; }
/* * Items needed: * mr.PoolId must be set * mr.StorageId should also be set * mr.ScratchPoolId could be set (used if create==true) * jcr->wstore * jcr->db * jcr->pool * MEDIA_DBR mr with PoolId set * create -- whether or not to create a new volume */ int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index, bool create, bool prune) { int retry = 0; bool ok; bool InChanger; STORE *store = jcr->wstore; bstrncpy(mr->MediaType, store->media_type, sizeof(mr->MediaType)); Dmsg3(100, "find_next_vol_for_append: JobId=%u PoolId=%d, MediaType=%s\n", (uint32_t)jcr->JobId, (int)mr->PoolId, mr->MediaType); /* * If we are using an Autochanger, restrict Volume * search to the Autochanger on the first pass */ InChanger = store->autochanger; /* * Find the Next Volume for Append */ db_lock(jcr->db); for ( ;; ) { bstrncpy(mr->VolStatus, "Append", sizeof(mr->VolStatus)); /* want only appendable volumes */ /* * 1. Look for volume with "Append" status. */ ok = db_find_next_volume(jcr, jcr->db, index, InChanger, mr); if (!ok) { Dmsg4(150, "after find_next_vol ok=%d index=%d InChanger=%d Vstat=%s\n", ok, index, InChanger, mr->VolStatus); /* * 2. Try finding a recycled volume */ ok = find_recycled_volume(jcr, InChanger, mr); Dmsg2(150, "find_recycled_volume ok=%d FW=%d\n", ok, mr->FirstWritten); if (!ok) { /* * 3. Try recycling any purged volume */ ok = recycle_oldest_purged_volume(jcr, InChanger, mr); if (!ok) { /* * 4. Try pruning Volumes */ if (prune) { Dmsg0(150, "Call prune_volumes\n"); prune_volumes(jcr, InChanger, mr); } ok = recycle_oldest_purged_volume(jcr, InChanger, mr); if (!ok && create) { Dmsg4(150, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n", ok, index, InChanger, mr->VolStatus); /* * 5. Try pulling a volume from the Scratch pool */ ok = get_scratch_volume(jcr, InChanger, mr); Dmsg4(150, "after get scratch volume ok=%d index=%d InChanger=%d Vstat=%s\n", ok, index, InChanger, mr->VolStatus); } /* * If we are using an Autochanger and have not found * a volume, retry looking for any volume. */ if (!ok && InChanger) { InChanger = false; continue; /* retry again accepting any volume */ } } } if (!ok && create) { /* * 6. Try "creating" a new Volume */ ok = newVolume(jcr, mr); } /* * Look at more drastic ways to find an Appendable Volume */ if (!ok && (jcr->pool->purge_oldest_volume || jcr->pool->recycle_oldest_volume)) { Dmsg2(200, "No next volume found. PurgeOldest=%d\n RecyleOldest=%d", jcr->pool->purge_oldest_volume, jcr->pool->recycle_oldest_volume); /* Find oldest volume to recycle */ ok = db_find_next_volume(jcr, jcr->db, -1, InChanger, mr); Dmsg1(200, "Find oldest=%d Volume\n", ok); if (ok && prune) { UAContext *ua; Dmsg0(200, "Try purge Volume.\n"); /* * 7. Try to purging oldest volume only if not UA calling us. */ ua = new_ua_context(jcr); if (jcr->pool->purge_oldest_volume && create) { Jmsg(jcr, M_INFO, 0, _("Purging oldest volume \"%s\"\n"), mr->VolumeName); ok = purge_jobs_from_volume(ua, mr); /* * 8. or try recycling the oldest volume */ } else if (jcr->pool->recycle_oldest_volume) { Jmsg(jcr, M_INFO, 0, _("Pruning oldest volume \"%s\"\n"), mr->VolumeName); ok = prune_volume(ua, mr); } free_ua_context(ua); if (ok) { ok = recycle_volume(jcr, mr); Dmsg1(400, "Recycle after purge oldest=%d\n", ok); } } } } Dmsg2(100, "VolJobs=%d FirstWritten=%d\n", mr->VolJobs, mr->FirstWritten); if (ok) { /* If we can use the volume, check if it is expired */ if (has_volume_expired(jcr, mr)) { if (retry++ < 200) { /* sanity check */ continue; /* try again from the top */ } else { Jmsg(jcr, M_ERROR, 0, _( "We seem to be looping trying to find the next volume. I give up.\n")); } } } break; } /* end for loop */ db_unlock(jcr->db); Dmsg1(150, "return ok=%d find_next_vol\n", ok); return ok; }
/* * Prune at least one Volume in current Pool. This is called from * catreq.c => next_vol.c when the Storage daemon is asking for another * volume and no appendable volumes are available. * */ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr, STORERES *store) { int i; int count; POOL_DBR spr; UAContext *ua; dbid_list ids; struct del_ctx prune_list; POOL_MEM query(PM_MESSAGE); char ed1[50], ed2[100], ed3[50]; Dmsg1(100, "Prune volumes PoolId=%d\n", jcr->jr.PoolId); if (!jcr->res.job->PruneVolumes && !jcr->res.pool->AutoPrune) { Dmsg0(100, "AutoPrune not set in Pool.\n"); return; } memset(&prune_list, 0, sizeof(prune_list)); prune_list.max_ids = 10000; prune_list.JobId = (JobId_t *)malloc(sizeof(JobId_t) * prune_list.max_ids); ua = new_ua_context(jcr); db_lock(jcr->db); /* Edit PoolId */ edit_int64(mr->PoolId, ed1); /* * Get Pool record for Scratch Pool */ memset(&spr, 0, sizeof(spr)); bstrncpy(spr.Name, "Scratch", sizeof(spr.Name)); if (db_get_pool_record(jcr, jcr->db, &spr)) { edit_int64(spr.PoolId, ed2); bstrncat(ed2, ",", sizeof(ed2)); } else { ed2[0] = 0; } if (mr->ScratchPoolId) { edit_int64(mr->ScratchPoolId, ed3); bstrncat(ed2, ed3, sizeof(ed2)); bstrncat(ed2, ",", sizeof(ed2)); } Dmsg1(100, "Scratch pool(s)=%s\n", ed2); /* * ed2 ends up with scratch poolid and current poolid or * just current poolid if there is no scratch pool */ bstrncat(ed2, ed1, sizeof(ed2)); /* * Get the List of all media ids in the current Pool or whose * RecyclePoolId is the current pool or the scratch pool */ const char *select = "SELECT DISTINCT MediaId,LastWritten FROM Media WHERE " "(PoolId=%s OR RecyclePoolId IN (%s)) AND MediaType='%s' %s" "ORDER BY LastWritten ASC,MediaId"; if (InChanger) { char changer[100]; /* Ensure it is in this autochanger */ bsnprintf(changer, sizeof(changer), "AND InChanger=1 AND StorageId=%s ", edit_int64(mr->StorageId, ed3)); Mmsg(query, select, ed1, ed2, mr->MediaType, changer); } else { Mmsg(query, select, ed1, ed2, mr->MediaType, ""); } Dmsg1(100, "query=%s\n", query.c_str()); if (!db_get_query_dbids(ua->jcr, ua->db, query, ids)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db)); goto bail_out; } Dmsg1(100, "Volume prune num_ids=%d\n", ids.num_ids); /* Visit each Volume and Prune it until we find one that is purged */ for (i=0; i<ids.num_ids; i++) { MEDIA_DBR lmr; lmr.MediaId = ids.DBId[i]; Dmsg1(100, "Get record MediaId=%d\n", (int)lmr.MediaId); if (!db_get_media_record(jcr, jcr->db, &lmr)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db)); continue; } Dmsg1(100, "Examine vol=%s\n", lmr.VolumeName); /* Don't prune archived volumes */ if (lmr.Enabled == 2) { Dmsg1(100, "Vol=%s disabled\n", lmr.VolumeName); continue; } /* Prune only Volumes with status "Full", or "Used" */ if (bstrcmp(lmr.VolStatus, "Full") || bstrcmp(lmr.VolStatus, "Used")) { Dmsg2(100, "Add prune list MediaId=%d Volume %s\n", (int)lmr.MediaId, lmr.VolumeName); count = get_prune_list_for_volume(ua, &lmr, &prune_list); Dmsg1(100, "Num pruned = %d\n", count); if (count != 0) { purge_job_list_from_catalog(ua, prune_list); prune_list.num_ids = 0; /* reset count */ } if (!is_volume_purged(ua, &lmr)) { Dmsg1(050, "Vol=%s not pruned\n", lmr.VolumeName); continue; } Dmsg1(050, "Vol=%s is purged\n", lmr.VolumeName); /* * Since we are also pruning the Scratch pool, continue * until and check if this volume is available (InChanger + StorageId) * If not, just skip this volume and try the next one */ if (InChanger) { if (!lmr.InChanger || (lmr.StorageId != mr->StorageId)) { Dmsg1(100, "Vol=%s not inchanger or correct StoreId\n", lmr.VolumeName); continue; /* skip this volume, ie not loadable */ } } if (!lmr.Recycle) { Dmsg1(100, "Vol=%s not recyclable\n", lmr.VolumeName); continue; } if (has_volume_expired(jcr, &lmr)) { Dmsg1(100, "Vol=%s has expired\n", lmr.VolumeName); continue; /* Volume not usable */ } /* * If purged and not moved to another Pool, * then we stop pruning and take this volume. */ if (lmr.PoolId == mr->PoolId) { Dmsg2(100, "Got Vol=%s MediaId=%d purged.\n", lmr.VolumeName, (int)lmr.MediaId); mr->copy(&lmr); set_storageid_in_mr(store, mr); break; /* got a volume */ } } } bail_out: Dmsg0(100, "Leave prune volumes\n"); db_unlock(jcr->db); free_ua_context(ua); if (prune_list.JobId) { free(prune_list.JobId); } return; }
/* * Try hard to recycle the current volume * * Returns: on failure - reason = NULL * on success - reason - pointer to reason */ void check_if_volume_valid_or_recyclable(JCR *jcr, MEDIA_DBR *mr, const char **reason) { int ok; *reason = NULL; if (!mr->Recycle) { *reason = _("volume has recycling disabled"); return; } /* Check if a duration or limit has expired */ if (has_volume_expired(jcr, mr)) { *reason = _("volume has expired"); /* Keep going because we may be able to recycle volume */ } /* * Now see if we can use the volume as is */ if (strcmp(mr->VolStatus, "Append") == 0 || strcmp(mr->VolStatus, "Recycle") == 0) { *reason = NULL; return; } /* * Check if the Volume is already marked for recycling */ if (strcmp(mr->VolStatus, "Purged") == 0) { if (recycle_volume(jcr, mr)) { Jmsg(jcr, M_INFO, 0, _("Recycled current volume \"%s\"\n"), mr->VolumeName); *reason = NULL; return; } else { /* In principle this shouldn't happen */ *reason = _("and recycling of current volume failed"); return; } } /* At this point, the volume is not valid for writing */ *reason = _("but should be Append, Purged or Recycle"); /* * What we're trying to do here is see if the current volume is * "recyclable" - ie. if we prune all expired jobs off it, is * it now possible to reuse it for the job that it is currently * needed for? */ if ((mr->LastWritten + mr->VolRetention) < (utime_t)time(NULL) && mr->Recycle && jcr->pool->recycle_current_volume && (strcmp(mr->VolStatus, "Full") == 0 || strcmp(mr->VolStatus, "Used") == 0)) { /* * Attempt prune of current volume to see if we can * recycle it for use. */ UAContext *ua; ua = new_ua_context(jcr); ok = prune_volume(ua, mr); free_ua_context(ua); if (ok) { /* If fully purged, recycle current volume */ if (recycle_volume(jcr, mr)) { Jmsg(jcr, M_INFO, 0, _("Recycled current volume \"%s\"\n"), mr->VolumeName); *reason = NULL; } else { *reason = _("but should be Append, Purged or Recycle (recycling of the " "current volume failed)"); } } else { *reason = _("but should be Append, Purged or Recycle (cannot automatically " "recycle current volume, as it still contains unpruned data " "or the Volume Retention time has not expired.)"); } } }