void catalog_request(JCR *jcr, BSOCK *bs)
{
MEDIA_DBR mr, sdmr;
JOBMEDIA_DBR jm;
char Job[MAX_NAME_LENGTH];
char pool_name[MAX_NAME_LENGTH];
int index, ok, label, writing;
POOLMEM *omsg;
POOL_DBR pr;
uint32_t Stripe, Copy;
uint64_t MediaId;
utime_t VolFirstWritten;
utime_t VolLastWritten;
memset(&sdmr, 0, sizeof(sdmr));
memset(&jm, 0, sizeof(jm));
Dsm_check(100);
/*
* Request to find next appendable Volume for this Job
*/
Dmsg1(100, "catreq %s", bs->msg);
if (!jcr->db) {
omsg = get_memory(bs->msglen+1);
pm_strcpy(omsg, bs->msg);
bs->fsend(_("1990 Invalid Catalog Request: %s"), omsg);
Jmsg1(jcr, M_FATAL, 0, _("Invalid Catalog request; DB not open: %s"), omsg);
free_memory(omsg);
return;
}
/*
* Find next appendable medium for SD
*/
if (sscanf(bs->msg, Find_media, &Job, &index, &pool_name, &mr.MediaType) == 4) {
memset(&pr, 0, sizeof(pr));
bstrncpy(pr.Name, pool_name, sizeof(pr.Name));
unbash_spaces(pr.Name);
ok = db_get_pool_record(jcr, jcr->db, &pr);
if (ok) {
mr.PoolId = pr.PoolId;
set_storageid_in_mr(jcr->res.wstore, &mr);
mr.ScratchPoolId = pr.ScratchPoolId;
ok = find_next_volume_for_append(jcr, &mr, index, fnv_create_vol, fnv_prune);
Dmsg3(050, "find_media ok=%d idx=%d vol=%s\n", ok, index, mr.VolumeName);
}
/*
* Send Find Media response to Storage daemon
*/
if (ok) {
send_volume_info_to_storage_daemon(jcr, bs, &mr);
} else {
bs->fsend(_("1901 No Media.\n"));
Dmsg0(500, "1901 No Media.\n");
}
/*
* Request to find specific Volume information
*/
} else if (sscanf(bs->msg, Get_Vol_Info, &Job, &mr.VolumeName, &writing) == 3) {
Dmsg1(100, "CatReq GetVolInfo Vol=%s\n", mr.VolumeName);
/*
* Find the Volume
*/
unbash_spaces(mr.VolumeName);
if (db_get_media_record(jcr, jcr->db, &mr)) {
const char *reason = NULL; /* detailed reason for rejection */
/*
* If we are reading, accept any volume (reason == NULL)
* If we are writing, check if the Volume is valid
* for this job, and do a recycle if necessary
*/
if (writing) {
/*
* SD wants to write this Volume, so make
* sure it is suitable for this job, i.e.
* Pool matches, and it is either Append or Recycle
* and Media Type matches and Pool allows any volume.
*/
if (mr.PoolId != jcr->jr.PoolId) {
reason = _("not in Pool");
} else if (!bstrcmp(mr.MediaType, jcr->res.wstore->media_type)) {
reason = _("not correct MediaType");
} else {
/*
* Now try recycling if necessary
* reason set non-NULL if we cannot use it
*/
check_if_volume_valid_or_recyclable(jcr, &mr, &reason);
}
}
if (!reason && mr.Enabled != 1) {
reason = _("is not Enabled");
}
if (reason == NULL) {
/*
* Send Find Media response to Storage daemon
*/
send_volume_info_to_storage_daemon(jcr, bs, &mr);
} else {
/* Not suitable volume */
bs->fsend(_("1998 Volume \"%s\" catalog status is %s, %s.\n"), mr.VolumeName,
mr.VolStatus, reason);
}
} else {
bs->fsend(_("1997 Volume \"%s\" not in catalog.\n"), mr.VolumeName);
Dmsg1(100, "1997 Volume \"%s\" not in catalog.\n", mr.VolumeName);
}
/*
* Request to update Media record. Comes typically at the end
* of a Storage daemon Job Session, when labeling/relabeling a
* Volume, or when an EOF mark is written.
*/
} else if (sscanf(bs->msg, Update_media, &Job, &sdmr.VolumeName,
&sdmr.VolJobs, &sdmr.VolFiles, &sdmr.VolBlocks, &sdmr.VolBytes,
&sdmr.VolMounts, &sdmr.VolErrors, &sdmr.VolWrites, &sdmr.MaxVolBytes,
&VolLastWritten, &sdmr.VolStatus, &sdmr.Slot, &label, &sdmr.InChanger,
&sdmr.VolReadTime, &sdmr.VolWriteTime, &VolFirstWritten) == 18) {
db_lock(jcr->db);
Dmsg3(400, "Update media %s oldStat=%s newStat=%s\n", sdmr.VolumeName,
mr.VolStatus, sdmr.VolStatus);
bstrncpy(mr.VolumeName, sdmr.VolumeName, sizeof(mr.VolumeName)); /* copy Volume name */
unbash_spaces(mr.VolumeName);
if (!db_get_media_record(jcr, jcr->db, &mr)) {
Jmsg(jcr, M_ERROR, 0, _("Unable to get Media record for Volume %s: ERR=%s\n"),
mr.VolumeName, db_strerror(jcr->db));
bs->fsend(_("1991 Catalog Request for vol=%s failed: %s"),
mr.VolumeName, db_strerror(jcr->db));
goto bail_out;
}
/* Set first written time if this is first job */
if (mr.FirstWritten == 0) {
if (VolFirstWritten == 0) {
mr.FirstWritten = jcr->start_time; /* use Job start time as first write */
} else {
mr.FirstWritten = VolFirstWritten;
}
mr.set_first_written = true;
}
/* If we just labeled the tape set time */
if (label || mr.LabelDate == 0) {
mr.LabelDate = jcr->start_time;
mr.set_label_date = true;
if (mr.InitialWrite == 0) {
mr.InitialWrite = jcr->start_time;
}
Dmsg2(400, "label=%d labeldate=%d\n", label, mr.LabelDate);
} else {
/*
* Insanity check for VolFiles get set to a smaller value
*/
if (sdmr.VolFiles < mr.VolFiles) {
Jmsg(jcr, M_FATAL, 0, _("Volume Files at %u being set to %u"
" for Volume \"%s\". This is incorrect.\n"),
mr.VolFiles, sdmr.VolFiles, mr.VolumeName);
bs->fsend(_("1992 Update Media error. VolFiles=%u, CatFiles=%u\n"),
sdmr.VolFiles, mr.VolFiles);
goto bail_out;
}
}
Dmsg2(400, "Update media: BefVolJobs=%u After=%u\n", mr.VolJobs, sdmr.VolJobs);
/*
* Check if the volume has been written by the job,
* and update the LastWritten field if needed.
*/
if (mr.VolBlocks != sdmr.VolBlocks && VolLastWritten != 0) {
mr.LastWritten = VolLastWritten;
}
/*
* Update to point to the last device used to write the Volume.
* However, do so only if we are writing the tape, i.e.
* the number of VolWrites has increased.
*/
if (jcr->res.wstore && sdmr.VolWrites > mr.VolWrites) {
Dmsg2(050, "Update StorageId old=%d new=%d\n",
mr.StorageId, jcr->res.wstore->StorageId);
/* Update StorageId after write */
set_storageid_in_mr(jcr->res.wstore, &mr);
} else {
/* Nothing written, reset same StorageId */
set_storageid_in_mr(NULL, &mr);
}
/* Copy updated values to original media record */
mr.VolJobs = sdmr.VolJobs;
mr.VolFiles = sdmr.VolFiles;
mr.VolBlocks = sdmr.VolBlocks;
mr.VolBytes = sdmr.VolBytes;
mr.VolMounts = sdmr.VolMounts;
mr.VolErrors = sdmr.VolErrors;
mr.VolWrites = sdmr.VolWrites;
mr.Slot = sdmr.Slot;
mr.InChanger = sdmr.InChanger;
bstrncpy(mr.VolStatus, sdmr.VolStatus, sizeof(mr.VolStatus));
mr.VolReadTime = sdmr.VolReadTime;
mr.VolWriteTime = sdmr.VolWriteTime;
Dmsg2(400, "db_update_media_record. Stat=%s Vol=%s\n", mr.VolStatus, mr.VolumeName);
/*
* Update the database, then before sending the response to the
* SD, check if the Volume has expired.
*/
if (!db_update_media_record(jcr, jcr->db, &mr)) {
Jmsg(jcr, M_FATAL, 0, _("Catalog error updating Media record. %s"),
db_strerror(jcr->db));
bs->fsend(_("1993 Update Media error\n"));
Dmsg0(400, "send error\n");
} else {
(void)has_volume_expired(jcr, &mr);
send_volume_info_to_storage_daemon(jcr, bs, &mr);
}
bail_out:
db_unlock(jcr->db);
Dmsg1(400, ">CatReq response: %s", bs->msg);
Dmsg1(400, "Leave catreq jcr 0x%x\n", jcr);
return;
/*
* Request to create a JobMedia record
*/
} else if (sscanf(bs->msg, Create_job_media, &Job,
&jm.FirstIndex, &jm.LastIndex, &jm.StartFile, &jm.EndFile,
&jm.StartBlock, &jm.EndBlock, &Copy, &Stripe, &MediaId) == 10) {
if (jcr->mig_jcr) {
jm.JobId = jcr->mig_jcr->JobId;
} else {
jm.JobId = jcr->JobId;
}
jm.MediaId = MediaId;
Dmsg6(400, "create_jobmedia JobId=%d MediaId=%d SF=%d EF=%d FI=%d LI=%d\n",
jm.JobId, jm.MediaId, jm.StartFile, jm.EndFile, jm.FirstIndex, jm.LastIndex);
if (!db_create_jobmedia_record(jcr, jcr->db, &jm)) {
Jmsg(jcr, M_FATAL, 0, _("Catalog error creating JobMedia record. %s"),
db_strerror(jcr->db));
bs->fsend(_("1992 Create JobMedia error\n"));
} else {
Dmsg0(400, "JobMedia record created\n");
bs->fsend(OK_create);
}
} else {
omsg = get_memory(bs->msglen+1);
pm_strcpy(omsg, bs->msg);
bs->fsend(_("1990 Invalid Catalog Request: %s"), omsg);
Jmsg1(jcr, M_FATAL, 0, _("Invalid Catalog request: %s"), omsg);
free_memory(omsg);
}
Dmsg1(400, ">CatReq response: %s", bs->msg);
Dmsg1(400, "Leave catreq jcr 0x%x\n", jcr);
return;
}
/*
* Items needed:
* mr.PoolId must be set
* mr.StorageId should also be set
* mr.ScratchPoolId could be set (used if create==true)
* jcr->wstore
* jcr->db
* jcr->pool
* MEDIA_DBR mr with PoolId set
* create -- whether or not to create a new volume
*/
int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index,
bool create, bool prune)
{
int retry = 0;
bool ok;
bool InChanger;
STORE *store = jcr->wstore;
bstrncpy(mr->MediaType, store->media_type, sizeof(mr->MediaType));
Dmsg3(100, "find_next_vol_for_append: JobId=%u PoolId=%d, MediaType=%s\n",
(uint32_t)jcr->JobId, (int)mr->PoolId, mr->MediaType);
/*
* If we are using an Autochanger, restrict Volume
* search to the Autochanger on the first pass
*/
InChanger = store->autochanger;
/*
* Find the Next Volume for Append
*/
db_lock(jcr->db);
for ( ;; ) {
bstrncpy(mr->VolStatus, "Append", sizeof(mr->VolStatus)); /* want only appendable volumes */
/*
* 1. Look for volume with "Append" status.
*/
ok = db_find_next_volume(jcr, jcr->db, index, InChanger, mr);
if (!ok) {
Dmsg4(150, "after find_next_vol ok=%d index=%d InChanger=%d Vstat=%s\n",
ok, index, InChanger, mr->VolStatus);
/*
* 2. Try finding a recycled volume
*/
ok = find_recycled_volume(jcr, InChanger, mr);
Dmsg2(150, "find_recycled_volume ok=%d FW=%d\n", ok, mr->FirstWritten);
if (!ok) {
/*
* 3. Try recycling any purged volume
*/
ok = recycle_oldest_purged_volume(jcr, InChanger, mr);
if (!ok) {
/*
* 4. Try pruning Volumes
*/
if (prune) {
Dmsg0(150, "Call prune_volumes\n");
prune_volumes(jcr, InChanger, mr);
}
ok = recycle_oldest_purged_volume(jcr, InChanger, mr);
if (!ok && create) {
Dmsg4(150, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n",
ok, index, InChanger, mr->VolStatus);
/*
* 5. Try pulling a volume from the Scratch pool
*/
ok = get_scratch_volume(jcr, InChanger, mr);
Dmsg4(150, "after get scratch volume ok=%d index=%d InChanger=%d Vstat=%s\n",
ok, index, InChanger, mr->VolStatus);
}
/*
* If we are using an Autochanger and have not found
* a volume, retry looking for any volume.
*/
if (!ok && InChanger) {
InChanger = false;
continue; /* retry again accepting any volume */
}
}
}
if (!ok && create) {
/*
* 6. Try "creating" a new Volume
*/
ok = newVolume(jcr, mr);
}
/*
* Look at more drastic ways to find an Appendable Volume
*/
if (!ok && (jcr->pool->purge_oldest_volume ||
jcr->pool->recycle_oldest_volume)) {
Dmsg2(200, "No next volume found. PurgeOldest=%d\n RecyleOldest=%d",
jcr->pool->purge_oldest_volume, jcr->pool->recycle_oldest_volume);
/* Find oldest volume to recycle */
ok = db_find_next_volume(jcr, jcr->db, -1, InChanger, mr);
Dmsg1(200, "Find oldest=%d Volume\n", ok);
if (ok && prune) {
UAContext *ua;
Dmsg0(200, "Try purge Volume.\n");
/*
* 7. Try to purging oldest volume only if not UA calling us.
*/
ua = new_ua_context(jcr);
if (jcr->pool->purge_oldest_volume && create) {
Jmsg(jcr, M_INFO, 0, _("Purging oldest volume \"%s\"\n"), mr->VolumeName);
ok = purge_jobs_from_volume(ua, mr);
/*
* 8. or try recycling the oldest volume
*/
} else if (jcr->pool->recycle_oldest_volume) {
Jmsg(jcr, M_INFO, 0, _("Pruning oldest volume \"%s\"\n"), mr->VolumeName);
ok = prune_volume(ua, mr);
}
free_ua_context(ua);
if (ok) {
ok = recycle_volume(jcr, mr);
Dmsg1(400, "Recycle after purge oldest=%d\n", ok);
}
}
}
}
Dmsg2(100, "VolJobs=%d FirstWritten=%d\n", mr->VolJobs, mr->FirstWritten);
if (ok) {
/* If we can use the volume, check if it is expired */
if (has_volume_expired(jcr, mr)) {
if (retry++ < 200) { /* sanity check */
continue; /* try again from the top */
} else {
Jmsg(jcr, M_ERROR, 0, _(
"We seem to be looping trying to find the next volume. I give up.\n"));
}
}
}
break;
} /* end for loop */
db_unlock(jcr->db);
Dmsg1(150, "return ok=%d find_next_vol\n", ok);
return ok;
}
/*
* Prune at least one Volume in current Pool. This is called from
* catreq.c => next_vol.c when the Storage daemon is asking for another
* volume and no appendable volumes are available.
*
*/
void prune_volumes(JCR *jcr, bool InChanger,
MEDIA_DBR *mr, STORERES *store)
{
int i;
int count;
POOL_DBR spr;
UAContext *ua;
dbid_list ids;
struct del_ctx prune_list;
POOL_MEM query(PM_MESSAGE);
char ed1[50], ed2[100], ed3[50];
Dmsg1(100, "Prune volumes PoolId=%d\n", jcr->jr.PoolId);
if (!jcr->res.job->PruneVolumes && !jcr->res.pool->AutoPrune) {
Dmsg0(100, "AutoPrune not set in Pool.\n");
return;
}
memset(&prune_list, 0, sizeof(prune_list));
prune_list.max_ids = 10000;
prune_list.JobId = (JobId_t *)malloc(sizeof(JobId_t) * prune_list.max_ids);
ua = new_ua_context(jcr);
db_lock(jcr->db);
/* Edit PoolId */
edit_int64(mr->PoolId, ed1);
/*
* Get Pool record for Scratch Pool
*/
memset(&spr, 0, sizeof(spr));
bstrncpy(spr.Name, "Scratch", sizeof(spr.Name));
if (db_get_pool_record(jcr, jcr->db, &spr)) {
edit_int64(spr.PoolId, ed2);
bstrncat(ed2, ",", sizeof(ed2));
} else {
ed2[0] = 0;
}
if (mr->ScratchPoolId) {
edit_int64(mr->ScratchPoolId, ed3);
bstrncat(ed2, ed3, sizeof(ed2));
bstrncat(ed2, ",", sizeof(ed2));
}
Dmsg1(100, "Scratch pool(s)=%s\n", ed2);
/*
* ed2 ends up with scratch poolid and current poolid or
* just current poolid if there is no scratch pool
*/
bstrncat(ed2, ed1, sizeof(ed2));
/*
* Get the List of all media ids in the current Pool or whose
* RecyclePoolId is the current pool or the scratch pool
*/
const char *select = "SELECT DISTINCT MediaId,LastWritten FROM Media WHERE "
"(PoolId=%s OR RecyclePoolId IN (%s)) AND MediaType='%s' %s"
"ORDER BY LastWritten ASC,MediaId";
if (InChanger) {
char changer[100];
/* Ensure it is in this autochanger */
bsnprintf(changer, sizeof(changer), "AND InChanger=1 AND StorageId=%s ",
edit_int64(mr->StorageId, ed3));
Mmsg(query, select, ed1, ed2, mr->MediaType, changer);
} else {
Mmsg(query, select, ed1, ed2, mr->MediaType, "");
}
Dmsg1(100, "query=%s\n", query.c_str());
if (!db_get_query_dbids(ua->jcr, ua->db, query, ids)) {
Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db));
goto bail_out;
}
Dmsg1(100, "Volume prune num_ids=%d\n", ids.num_ids);
/* Visit each Volume and Prune it until we find one that is purged */
for (i=0; i<ids.num_ids; i++) {
MEDIA_DBR lmr;
lmr.MediaId = ids.DBId[i];
Dmsg1(100, "Get record MediaId=%d\n", (int)lmr.MediaId);
if (!db_get_media_record(jcr, jcr->db, &lmr)) {
Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db));
continue;
}
Dmsg1(100, "Examine vol=%s\n", lmr.VolumeName);
/* Don't prune archived volumes */
if (lmr.Enabled == 2) {
Dmsg1(100, "Vol=%s disabled\n", lmr.VolumeName);
continue;
}
/* Prune only Volumes with status "Full", or "Used" */
if (bstrcmp(lmr.VolStatus, "Full") ||
bstrcmp(lmr.VolStatus, "Used")) {
Dmsg2(100, "Add prune list MediaId=%d Volume %s\n", (int)lmr.MediaId, lmr.VolumeName);
count = get_prune_list_for_volume(ua, &lmr, &prune_list);
Dmsg1(100, "Num pruned = %d\n", count);
if (count != 0) {
purge_job_list_from_catalog(ua, prune_list);
prune_list.num_ids = 0; /* reset count */
}
if (!is_volume_purged(ua, &lmr)) {
Dmsg1(050, "Vol=%s not pruned\n", lmr.VolumeName);
continue;
}
Dmsg1(050, "Vol=%s is purged\n", lmr.VolumeName);
/*
* Since we are also pruning the Scratch pool, continue
* until and check if this volume is available (InChanger + StorageId)
* If not, just skip this volume and try the next one
*/
if (InChanger) {
if (!lmr.InChanger || (lmr.StorageId != mr->StorageId)) {
Dmsg1(100, "Vol=%s not inchanger or correct StoreId\n", lmr.VolumeName);
continue; /* skip this volume, ie not loadable */
}
}
if (!lmr.Recycle) {
Dmsg1(100, "Vol=%s not recyclable\n", lmr.VolumeName);
continue;
}
if (has_volume_expired(jcr, &lmr)) {
Dmsg1(100, "Vol=%s has expired\n", lmr.VolumeName);
continue; /* Volume not usable */
}
/*
* If purged and not moved to another Pool,
* then we stop pruning and take this volume.
*/
if (lmr.PoolId == mr->PoolId) {
Dmsg2(100, "Got Vol=%s MediaId=%d purged.\n", lmr.VolumeName, (int)lmr.MediaId);
mr->copy(&lmr);
set_storageid_in_mr(store, mr);
break; /* got a volume */
}
}
}
bail_out:
Dmsg0(100, "Leave prune volumes\n");
db_unlock(jcr->db);
free_ua_context(ua);
if (prune_list.JobId) {
free(prune_list.JobId);
}
return;
}
/*
* Try hard to recycle the current volume
*
* Returns: on failure - reason = NULL
* on success - reason - pointer to reason
*/
void check_if_volume_valid_or_recyclable(JCR *jcr, MEDIA_DBR *mr, const char **reason)
{
int ok;
*reason = NULL;
if (!mr->Recycle) {
*reason = _("volume has recycling disabled");
return;
}
/* Check if a duration or limit has expired */
if (has_volume_expired(jcr, mr)) {
*reason = _("volume has expired");
/* Keep going because we may be able to recycle volume */
}
/*
* Now see if we can use the volume as is
*/
if (strcmp(mr->VolStatus, "Append") == 0 ||
strcmp(mr->VolStatus, "Recycle") == 0) {
*reason = NULL;
return;
}
/*
* Check if the Volume is already marked for recycling
*/
if (strcmp(mr->VolStatus, "Purged") == 0) {
if (recycle_volume(jcr, mr)) {
Jmsg(jcr, M_INFO, 0, _("Recycled current volume \"%s\"\n"), mr->VolumeName);
*reason = NULL;
return;
} else {
/* In principle this shouldn't happen */
*reason = _("and recycling of current volume failed");
return;
}
}
/* At this point, the volume is not valid for writing */
*reason = _("but should be Append, Purged or Recycle");
/*
* What we're trying to do here is see if the current volume is
* "recyclable" - ie. if we prune all expired jobs off it, is
* it now possible to reuse it for the job that it is currently
* needed for?
*/
if ((mr->LastWritten + mr->VolRetention) < (utime_t)time(NULL)
&& mr->Recycle && jcr->pool->recycle_current_volume
&& (strcmp(mr->VolStatus, "Full") == 0 ||
strcmp(mr->VolStatus, "Used") == 0)) {
/*
* Attempt prune of current volume to see if we can
* recycle it for use.
*/
UAContext *ua;
ua = new_ua_context(jcr);
ok = prune_volume(ua, mr);
free_ua_context(ua);
if (ok) {
/* If fully purged, recycle current volume */
if (recycle_volume(jcr, mr)) {
Jmsg(jcr, M_INFO, 0, _("Recycled current volume \"%s\"\n"), mr->VolumeName);
*reason = NULL;
} else {
*reason = _("but should be Append, Purged or Recycle (recycling of the "
"current volume failed)");
}
} else {
*reason = _("but should be Append, Purged or Recycle (cannot automatically "
"recycle current volume, as it still contains unpruned data "
"or the Volume Retention time has not expired.)");
}
}
}
