void media_identifier::identify_data(const char *name, const uint8_t *data, std::size_t length)
{
std::vector<file_info> info;
digest_data(info, name, data, length);
match_hashes(info);
print_results(info);
}
int main(int argc, char *argv[])
{
FILE *fd;
char buf[8192];
int buflen;
digestctx_t *ctx;
if (argc < 2) {
printf("Usage: %s digestmethod [filename]\n", argv[0]);
printf("\"digestmethod\" is \"md5\", \"sha1\", \"sha256\", \"sha512\" or \"rmd160\"\n");
return 1;
}
if ((ctx = digest_init(argv[1])) == NULL) {
printf("Unknown message digest method %s\n", argv[1]);
return 1;
}
if (argc > 2) fd = fopen(argv[2], "r"); else fd = stdin;
if (fd == NULL) {
printf("Cannot open file %s\n", argv[2]);
return 1;
}
while ((buflen = fread(buf, 1, sizeof(buf), fd)) > 0) {
digest_data(ctx, buf, buflen);
}
printf("%s\n", digest_done(ctx));
return 0;
}
int digest_rx_data(struct iscsi_cmnd *cmnd)
{
struct tio *tio;
struct iscsi_cmnd *scsi_cmnd;
struct iscsi_data_out_hdr *req;
u32 offset, crc;
switch (cmnd_opcode(cmnd)) {
case ISCSI_OP_SCSI_REJECT:
case ISCSI_OP_PDU_REJECT:
case ISCSI_OP_DATA_REJECT:
return 0;
case ISCSI_OP_SCSI_DATA_OUT:
scsi_cmnd = cmnd->req;
req = (struct iscsi_data_out_hdr *) &cmnd->pdu.bhs;
tio = scsi_cmnd->tio;
offset = be32_to_cpu(req->buffer_offset);
break;
default:
tio = cmnd->tio;
offset = 0;
}
digest_data(&cmnd->conn->rx_hash, cmnd, tio, offset, (u8 *) &crc);
if (!cmnd->conn->read_overflow &&
(cmnd_opcode(cmnd) != ISCSI_OP_PDU_REJECT)) {
if (crc != cmnd->ddigest)
return -EIO;
}
return 0;
}
void digest_tx_data(struct iscsi_cmnd *cmnd)
{
struct tio *tio = cmnd->tio;
struct iscsi_data_out_hdr *req = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
assert(tio);
digest_data(&cmnd->conn->tx_hash, cmnd, tio,
be32_to_cpu(req->buffer_offset), (u8 *) &cmnd->ddigest);
}
char *filesum(char *fn, char *dtype)
{
static char *result = NULL;
digestctx_t *ctx;
FILE *fd;
unsigned char buf[8192];
int openerr, buflen;
if ((ctx = digest_init(dtype)) == NULL) return "";
fd = fileopen(fn, &openerr);
if (fd == NULL) return "";
while ((buflen = fread(buf, 1, sizeof(buf), fd)) > 0) digest_data(ctx, buf, buflen);
fclose(fd);
if (result) xfree(result);
result = strdup(digest_done(ctx));
return result;
}
char *tls_data_fprint(const char *buf, int len, const char *mdalg)
{
EVP_MD_CTX *mdctx;
const EVP_MD *md;
unsigned char md_buf[EVP_MAX_MD_SIZE];
unsigned int md_len;
int ok = 1;
/* Previously available in "init" routine. */
if ((md = EVP_get_digestbyname(mdalg)) == 0)
msg_panic("digest algorithm \"%s\" not found", mdalg);
mdctx = EVP_MD_CTX_create();
checkok(EVP_DigestInit_ex(mdctx, md, NULL));
digest_data(buf, len);
checkok(EVP_DigestFinal_ex(mdctx, md_buf, &md_len));
EVP_MD_CTX_destroy(mdctx);
if (!ok)
msg_fatal("error computing %s message digest", mdalg);
return (tls_digest_encode(md_buf, md_len));
}
int tcp_http_data_callback(unsigned char *buf, unsigned int len, void *priv)
{
/*
* This callback receives data from HTTP servers.
* While doing that, it splits out the data into a
* buffer for the HTTP headers, and a buffer for the
* HTTP content-data.
* Return 1 if data is complete, 0 if more data wanted.
*/
http_data_t *item = (http_data_t *) priv;
if (item->gotheaders) {
unsigned int len1chunk = 0;
int i;
/*
* We already have the headers, so just stash away the data
*/
while (len > 0) {
dbgprintf("HDC IN : state=%d, leftinchunk=%d, len=%d\n", item->chunkstate, item->leftinchunk, len);
switch (item->chunkstate) {
case CHUNK_NOTCHUNKED:
len1chunk = len;
if ((item->contlen > 0) && (item->contlen >= len)) item->contlen -= len;
break;
case CHUNK_INIT:
/* We're about to pick up a chunk length */
item->leftinchunk = 0;
item->chunkstate = CHUNK_GETLEN;
len1chunk = 0;
break;
case CHUNK_GETLEN:
/* We are collecting the length of the chunk */
i = hexvalue(*buf);
if (i == -1) {
item->chunkstate = CHUNK_SKIPLENCR;
}
else {
item->leftinchunk = item->leftinchunk*16 + i;
buf++; len--;
}
len1chunk = 0;
break;
case CHUNK_SKIPLENCR:
/* We've got the length, now skip to the next LF */
if (*buf == '\n') {
buf++; len--;
item->chunkstate = ((item->leftinchunk > 0) ? CHUNK_DATA : CHUNK_NOMORE);
}
else if ((*buf == '\r') || (*buf == ' ')) {
buf++; len--;
}
else {
errprintf("Yikes - strange data following chunk len. Saw a '%c'\n", *buf);
buf++; len--;
}
len1chunk = 0;
break;
case CHUNK_DATA:
/* Passing off the data */
if (len > item->leftinchunk) len1chunk = item->leftinchunk;
else len1chunk = len;
item->leftinchunk -= len1chunk;
if (item->leftinchunk == 0) item->chunkstate = CHUNK_SKIPENDCR;
break;
case CHUNK_SKIPENDCR:
/* Skip CR/LF after a chunk */
if (*buf == '\n') {
buf++; len--; item->chunkstate = CHUNK_DONE;
}
else if (*buf == '\r') {
buf++; len--;
}
else {
errprintf("Yikes - strange data following chunk data. Saw a '%c'\n", *buf);
buf++; len--;
}
len1chunk = 0;
break;
case CHUNK_DONE:
/* One chunk is done, continue with the next */
len1chunk = 0;
item->chunkstate = CHUNK_GETLEN;
break;
case CHUNK_NOMORE:
/* All chunks done. Skip the rest (trailers) */
len1chunk = 0;
len = 0;
}
if (len1chunk > 0) {
switch (item->contentcheck) {
case CONTENTCHECK_NONE:
case CONTENTCHECK_CONTENTTYPE:
/* No need to save output - just drop it */
break;
case CONTENTCHECK_REGEX:
case CONTENTCHECK_NOREGEX:
/* Save the full data */
if ((item->output == NULL) || (item->outlen == 0)) {
item->output = (unsigned char *)malloc(len1chunk+1);
}
else {
item->output = (unsigned char *)realloc(item->output, item->outlen+len1chunk+1);
}
memcpy(item->output+item->outlen, buf, len1chunk);
item->outlen += len1chunk;
*(item->output + item->outlen) = '\0'; /* Just in case ... */
break;
case CONTENTCHECK_DIGEST:
/* Run the data through our digest routine, but discard the raw data */
if ((item->digestctx == NULL) || (digest_data(item->digestctx, buf, len1chunk) != 0)) {
errprintf("Failed to hash data for digest\n");
}
break;
}
buf += len1chunk;
len -= len1chunk;
dbgprintf("HDC OUT: state=%d, leftinchunk=%d, len=%d\n", item->chunkstate, item->leftinchunk, len);
}
}
}
else {
/*
* Havent seen the end of headers yet.
*/
unsigned char *p;
/* First, add this to the header-buffer */
if (item->headers == NULL) {
item->headers = (unsigned char *) malloc(len+1);
}
else {
item->headers = (unsigned char *) realloc(item->headers, item->hdrlen+len+1);
}
memcpy(item->headers+item->hdrlen, buf, len);
item->hdrlen += len;
*(item->headers + item->hdrlen) = '\0';
check_for_endofheaders:
/*
* Now see if we have the end-of-headers delimiter.
* This SHOULD be <cr><lf><cr><lf>, but RFC 2616 says
* you SHOULD recognize just plain <lf><lf>.
* So try the second form, if the first one is not there.
*/
p=strstr(item->headers, "\r\n\r\n");
if (p) {
p += 4;
}
else {
p = strstr(item->headers, "\n\n");
if (p) p += 2;
}
if (p) {
int http1subver, httpstatus;
unsigned int bytesindata;
char *p1, *xferencoding;
int contlen;
/* We have an end-of-header delimiter, but it could be just a "100 Continue" response */
sscanf(item->headers, "HTTP/1.%d %d", &http1subver, &httpstatus);
if (httpstatus == 100) {
/*
* It's a "100" continue-status.
* Just drop this set of headers, and move on.
*/
item->hdrlen -= (p - item->headers);
if (item->hdrlen > 0) {
memmove(item->headers, p, item->hdrlen);
*(item->headers + item->hdrlen) = '\0';
goto check_for_endofheaders;
}
else {
xfree(item->headers);
item->headers = NULL;
item->hdrlen = 0;
return 0;
}
/* Should never go here ... */
}
/* We did find the end-of-header delimiter, and it is not a "100" status. */
item->gotheaders = 1;
/* p points at the first byte of DATA. So the header ends 1 or 2 bytes before. */
*(p-1) = '\0';
if (*(p-2) == '\r') { *(p-2) = '\0'; } /* NULL-terminate the headers. */
/* See if the transfer uses chunks */
p1 = item->headers; xferencoding = NULL; contlen = 0;
do {
if (strncasecmp(p1, "Transfer-encoding:", 18) == 0) {
p1 += 18; while (isspace((int)*p1)) p1++;
xferencoding = p1;
}
else if (strncasecmp(p1, "Content-Length:", 15) == 0) {
p1 += 15; while (isspace((int)*p1)) p1++;
contlen = atoi(p1);
}
else {
p1 = strchr(p1, '\n'); if (p1) p1++;
}
} while (p1 && (xferencoding == NULL));
if (xferencoding && (strncasecmp(xferencoding, "chunked", 7) == 0)) {
item->chunkstate = CHUNK_INIT;
}
item->contlen = (contlen ? contlen : -1);
bytesindata = item->hdrlen - (p - item->headers);
item->hdrlen = strlen(item->headers);
if (*p) {
/*
* We received some content data together with the
* headers. Save these to the content-data area.
*/
tcp_http_data_callback(p, bytesindata, priv);
}
}
}
if (item->chunkstate == CHUNK_NOTCHUNKED)
/* Not chunked - we're done if contlen reaches 0 */
return (item->contlen == 0);
else
/* Chunked - we're done if we reach state NOMORE*/
return (item->chunkstate == CHUNK_NOMORE);
}
int main(int argc, char *argv[])
{
char srcfn[PATH_MAX];
char destfn[PATH_MAX];
char *p;
struct stat st;
unsigned char *sumbuf = NULL;
if (argc > 2) {
if (stat(argv[2], &st) == 0) {
FILE *sumfd;
printf("Loading md5-data ... ");
sumfd = fopen(argv[2], "r");
if (sumfd) {
sumbuf = (char *)malloc(st.st_size + 1);
if (fread(sumbuf, 1, st.st_size, sumfd) == st.st_size) {
printf("OK\n");
}
else {
printf("failed\n");
free(sumbuf);
sumbuf = NULL;
}
fclose(sumfd);
}
else {
printf("failed\n");
}
}
}
while (fgets(srcfn, sizeof(srcfn), stdin)) {
FILE *fd;
unsigned char buf[8192];
size_t buflen;
digestctx_t *ctx;
char srcmd5[40];
char *md5sum;
p = strchr(srcfn, '\n'); if (p) *p = '\0';
strcpy(destfn, argv[1]);
p = srcfn;
if (strcmp(srcfn, ".") == 0) p = "";
else if (strncmp(p, "./", 2) == 0) p += 2;
strcat(destfn, p);
*srcmd5 = '\0';
if (((fd = fopen(srcfn, "r")) != NULL) && ((ctx = digest_init("md5")) != NULL)) {
while ((buflen = fread(buf, 1, sizeof(buf), fd)) > 0) digest_data(ctx, buf, buflen);
strcpy(srcmd5, digest_done(ctx));
fclose(fd);
}
if (stat(destfn, &st) == 0) {
/* Destination file exists, see if it's a previous version */
if (sumbuf == NULL) continue; /* No md5-data, don't overwrite an existing file */
if (!S_ISREG(st.st_mode)) continue;
fd = fopen(destfn, "r"); if (fd == NULL) continue;
if ((ctx = digest_init("md5")) == NULL) continue;
while ((buflen = fread(buf, 1, sizeof(buf), fd)) > 0) digest_data(ctx, buf, buflen);
md5sum = digest_done(ctx);
fclose(fd);
if (strstr(sumbuf, md5sum) == NULL) continue; /* Not one of our known versions */
if (strcmp(srcmd5, md5sum) == 0) continue; /* Already installed */
/* We now know the destination that exists is just one of our old files */
printf("Updating old standard file %s\n", destfn);
unlink(destfn);
}
else {
printf("Installing new file %s\n", destfn);
}
if (lstat(srcfn, &st) != 0) {
printf("Error - cannot lstat() %s\n", srcfn);
return 1;
}
if (S_ISREG(st.st_mode)) {
FILE *infd, *outfd;
char buf[16384];
int n;
infd = fopen(srcfn, "r");
if (infd == NULL) {
/* Don't know how this can happen, but .. */
fprintf(stderr, "Cannot open input file %s: %s\n", srcfn, strerror(errno));
return 1;
}
outfd = fopen(destfn, "w");
if (outfd == NULL) {
/* Don't know how this can happen, but .. */
fprintf(stderr, "Cannot create output file %s: %s\n", destfn, strerror(errno));
return 1;
}
fchmod(fileno(outfd), S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
while ( (n = fread(buf, 1, sizeof(buf), infd)) > 0) fwrite(buf, 1, n, outfd);
fclose(infd); fclose(outfd);
}
else if (S_ISDIR(st.st_mode)) {
struct stat tmpst;
/* Create upper-lying directories */
if (*destfn == '/') p = strchr(destfn+1, '/'); else p = strchr(destfn, '/');
while (p) {
*p = '\0';
if ((stat(destfn, &tmpst) == 0) || (mkdir(destfn, st.st_mode) == 0)) {
*p = '/';
p = strchr(p+1, '/');
}
else p = NULL;
}
/* Create the directory itself */
if (stat(destfn, &tmpst) == -1) mkdir(destfn, st.st_mode);
chmod(destfn, S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
}
else if (S_ISLNK(st.st_mode)) {
char ldest[PATH_MAX + 1];
memset(ldest, 0, sizeof(ldest));
if ((readlink(srcfn, ldest, sizeof(ldest)-1) != -1) && (symlink(ldest, destfn) == 0)) {};
}
}
return 0;
}
void media_identifier::collect_files(std::vector<file_info> &info, char const *path)
{
// first try to open as a directory
osd::directory::ptr const directory = osd::directory::open(path);
if (directory)
{
// iterate over all files in the directory
for (osd::directory::entry const *entry = directory->read(); entry; entry = directory->read())
{
if (entry->type == osd::directory::entry::entry_type::FILE)
{
std::string const curfile = std::string(path).append(PATH_SEPARATOR).append(entry->name);
collect_files(info, curfile.c_str());
}
}
}
else if (core_filename_ends_with(path, ".7z") || core_filename_ends_with(path, ".zip"))
{
// first attempt to examine it as a valid zip/7z file
util::archive_file::ptr archive;
util::archive_file::error err;
if (core_filename_ends_with(path, ".7z"))
err = util::archive_file::open_7z(path, archive);
else
err = util::archive_file::open_zip(path, archive);
if ((util::archive_file::error::NONE == err) && archive)
{
std::vector<std::uint8_t> data;
// loop over entries in the .7z, skipping empty files and directories
for (int i = archive->first_file(); i >= 0; i = archive->next_file())
{
std::uint64_t const length(archive->current_uncompressed_length());
if (!archive->current_is_directory() && length)
{
std::string const curfile = std::string(path).append(PATH_SEPARATOR).append(archive->current_name());
if (std::uint32_t(length) == length)
{
// decompress data into RAM and identify it
try
{
data.resize(std::size_t(length));
err = archive->decompress(&data[0], std::uint32_t(length));
if (util::archive_file::error::NONE == err)
digest_data(info, curfile.c_str(), &data[0], length);
else
osd_printf_error("%s: error decompressing file\n", curfile.c_str());
}
catch (...)
{
// resizing the buffer could cause a bad_alloc if archive contains large files
osd_printf_error("%s: error decompressing file\n", curfile.c_str());
}
data.clear();
}
else
{
osd_printf_error("%s: file too large to decompress into memory\n", curfile.c_str());
}
}
}
}
else
{
osd_printf_error("%s: error opening archive\n", path);
}
// clear out any cached files
util::archive_file::cache_clear();
}
else
{
// otherwise, identify as a raw file
digest_file(info, path);
}
}