ByteVector zlib::decompress(const ByteVector &data) { #ifdef HAVE_ZLIB z_stream stream = {}; if(inflateInit(&stream) != Z_OK) { debug("zlib::decompress() - Failed to initizlize zlib."); return ByteVector(); } ByteVector inData = data; stream.avail_in = static_cast<uInt>(inData.size()); stream.next_in = reinterpret_cast<Bytef *>(inData.data()); const unsigned int chunkSize = 1024; ByteVector outData; do { const size_t offset = outData.size(); outData.resize(outData.size() + chunkSize); stream.avail_out = static_cast<uInt>(chunkSize); stream.next_out = reinterpret_cast<Bytef *>(outData.data() + offset); const int result = inflate(&stream, Z_NO_FLUSH); if(result == Z_STREAM_ERROR || result == Z_NEED_DICT || result == Z_DATA_ERROR || result == Z_MEM_ERROR) { if(result != Z_STREAM_ERROR) inflateEnd(&stream); debug("zlib::decompress() - Error reading compressed stream."); return ByteVector(); } outData.resize(outData.size() - stream.avail_out); } while(stream.avail_out == 0); inflateEnd(&stream); return outData; #else return ByteVector(); #endif }
virtual String parse(const ByteVector &data) const { std::string strSource(data.data(), data.size()); std::string strUTF8; g_charsetConverter.unknownToUTF8(strSource, strUTF8); return String(strUTF8, String::UTF8); }
ErrorCode File::pread(ByteVector &buf, uint64_t &count, uint64_t offset) { if (!valid()) { return _lastError = kErrorInvalidHandle; } if (offset > std::numeric_limits<off_t>::max()) { return _lastError = kErrorInvalidArgument; } auto offArg = static_cast<off_t>(offset); auto countArg = static_cast<size_t>(count); buf.resize(countArg); ssize_t nRead = ::pread(_fd, buf.data(), countArg, offArg); if (nRead < 0) { return _lastError = Platform::TranslateError(); } buf.resize(nRead); count = static_cast<uint64_t>(nRead); return _lastError = kSuccess; }
/** * Read meta information from APE tags * @param tag: the APE tag * @param p_demux_meta: the demuxer meta * @param p_meta: the meta */ static void ReadMetaFromAPE( APE::Tag* tag, demux_meta_t* p_demux_meta, vlc_meta_t* p_meta ) { APE::Item item; item = tag->itemListMap()["COVER ART (FRONT)"]; if( !item.isEmpty() ) { input_attachment_t *p_attachment; const ByteVector picture = item.value(); const char *p_data = picture.data(); unsigned i_data = picture.size(); size_t desc_len = strnlen(p_data, i_data); if (desc_len < i_data) { const char *psz_name = p_data; p_data += desc_len + 1; /* '\0' */ i_data -= desc_len + 1; msg_Dbg( p_demux_meta, "Found embedded art: %s (%s) is %u bytes", psz_name, "image/jpeg", i_data ); p_attachment = vlc_input_attachment_New( "cover", "image/jpeg", psz_name, p_data, i_data ); if( p_attachment ) TAB_APPEND_CAST( (input_attachment_t**), p_demux_meta->i_attachments, p_demux_meta->attachments, p_attachment ); vlc_meta_SetArtURL( p_meta, "attachment://cover" ); } }
ByteVector Frame::fieldData(const ByteVector &frameData) const { uint headerSize = Header::size(d->header->version()); uint frameDataOffset = headerSize; uint frameDataLength = size(); if(d->header->compression() || d->header->dataLengthIndicator()) { frameDataLength = SynchData::toUInt(frameData.mid(headerSize, 4)); frameDataOffset += 4; } #if HAVE_ZLIB if(d->header->compression() && !d->header->encryption()) { ByteVector data(frameDataLength); uLongf uLongTmp = frameDataLength; ::uncompress((Bytef *) data.data(), (uLongf *) &uLongTmp, (Bytef *) frameData.data() + frameDataOffset, size()); return data; } else #endif return frameData.mid(frameDataOffset, frameDataLength); }
virtual aku_MemRange allocate() { aku_MemRange range = { out->data(), static_cast<uint32_t>(out->size()) }; return range; }
ErrorCode File::pwrite(ByteVector const &buf, uint64_t &count, uint64_t offset) { DS2ASSERT(count > 0); if (!valid()) { return _lastError = kErrorInvalidHandle; } if (offset > std::numeric_limits<off_t>::max()) { return _lastError = kErrorInvalidArgument; } auto offArg = static_cast<off_t>(offset); auto countArg = static_cast<size_t>(count); ssize_t nWritten = ::pwrite(_fd, buf.data(), countArg, offArg); if (nWritten < 0) { return _lastError = Platform::TranslateError(); } count = static_cast<uint64_t>(nWritten); return _lastError = kSuccess; }
void ByteVectorStream::writeBlock(const ByteVector &data) { uint size = data.size(); if(d->position + size > length()) { truncate(d->position + size); } memcpy(d->data.data() + d->position, data.data(), size); d->position += size; }
void ByteVectorStream::writeBlock(const ByteVector &data) { const size_t size = data.size(); if(static_cast<long long>(d->position + size) > length()) truncate(d->position + size); ::memcpy(d->data.data() + d->position, data.data(), size); d->position += size; }
void RIFF::AIFF::Properties::read(const ByteVector &data) { d->channels = data.toShort(0U); d->sampleFrames = data.toUInt(2U); d->sampleWidth = data.toShort(6U); double sampleRate = ConvertFromIeeeExtended(reinterpret_cast<const uchar *>(data.data() + 8)); d->sampleRate = (int)sampleRate; d->bitrate = (int)((sampleRate * d->sampleWidth * d->channels) / 1000.0); d->length = d->sampleRate > 0 ? d->sampleFrames / d->sampleRate : 0; }
void File::writeBlock(const ByteVector &data) { if(!d->file) return; if(d->readOnly) { debug("File::writeBlock() -- attempted to write to a file that is not writable"); return; } fwrite(data.data(), sizeof(char), data.size(), d->file); }
String detectImageMimeType(const ByteVector &image) { if (image.size() > 4) { const unsigned char *data = (const unsigned char *)image.data(); if (data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF && data[3] == 0xE0) { return "image/jpeg"; } else if (data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47) { return "image/png"; } } return "application/octet-stream"; }
size_t OFile::write(const ByteVector &vector) { size_t cnt = 0, size; const char *buffer; if (stream == NULL) return 0; size = vector.size(); buffer = vector.data(); while (cnt < size && !ferror(stream)) cnt += fwrite(buffer, 1, size - cnt, stream); return cnt; }
void MP4::Tag::parseCovr(MP4::Atom *atom, TagLib::File *file) { MP4::CoverArtList value; ByteVector data = file->readBlock(atom->length - 8); unsigned int pos = 0; while(pos < data.size()) { int length = data.mid(pos, 4).toUInt(); ByteVector name = data.mid(pos + 4, 4); int flags = data.mid(pos + 8, 4).toUInt(); if(name != "data") { debug("MP4: Unexpected atom \"" + name + "\", expecting \"data\""); break; } if (flags == 0) { //detect cover format when the cover format bytes are not set ByteVector picHeader = data.mid(pos+16,9); const char jpeg[] = {0xff, 0xd8, 0xff, 0xe0 }; const char jfif[] = {0x10, 0x4a, 0x46, 0x49, 0x46 }; const char png[] = {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00 }; if ((memcmp(picHeader.data(), png, 9) == 0)) { flags = MP4::CoverArt::PNG; } else if ((memcmp(picHeader.data(), jpeg, 4) == 0)) { flags = MP4::CoverArt::JPEG; } else if ((memcmp(picHeader.data(), jfif, 5) == 0)) { flags = MP4::CoverArt::JPEG; } } if(flags == MP4::CoverArt::PNG || flags == MP4::CoverArt::JPEG || flags == 0) { value.append(MP4::CoverArt(MP4::CoverArt::Format(flags), data.mid(pos + 16, length - 16))); } pos += length; } if(value.size() > 0) d->items.insert(atom->name, value); }
///--------------------------------------------------------------------------------- /// ///--------------------------------------------------------------------------------- bool WriteByteVectorToFile( const std::string& filePath, const ByteVector& byteVector ) { FILE* file; fopen_s( &file, filePath.c_str(), "wb" ); if (!file) { //freak out return false; } fwrite( byteVector.data(), sizeof( unsigned char ), byteVector.size(), file ); fclose( file ); return true; }
/** * Read meta information from APE tags * @param tag: the APE tag * @param p_demux_meta: the demuxer meta * @param p_meta: the meta */ static void ReadMetaFromAPE( APE::Tag* tag, demux_meta_t* p_demux_meta, vlc_meta_t* p_meta ) { APE::ItemListMap fields ( tag->itemListMap() ); APE::ItemListMap::Iterator iter; iter = fields.find("COVER ART (FRONT)"); if( iter != fields.end() && !iter->second.isEmpty() && iter->second.type() == APE::Item::Binary) { input_attachment_t *p_attachment; const ByteVector picture = iter->second.binaryData(); const char *p_data = picture.data(); unsigned i_data = picture.size(); /* Null terminated filename followed by the image data */ size_t desc_len = strnlen(p_data, i_data); if( desc_len < i_data && IsUTF8( p_data ) ) { const char *psz_name = p_data; const char *psz_mime = vlc_mime_Ext2Mime( psz_name ); p_data += desc_len + 1; /* '\0' */ i_data -= desc_len + 1; msg_Dbg( p_demux_meta, "Found embedded art: %s (%s) is %u bytes", psz_name, psz_mime, i_data ); p_attachment = vlc_input_attachment_New( psz_name, psz_mime, psz_name, p_data, i_data ); if( p_attachment ) { TAB_APPEND_CAST( (input_attachment_t**), p_demux_meta->i_attachments, p_demux_meta->attachments, p_attachment ); char *psz_url; if( asprintf( &psz_url, "attachment://%s", p_attachment->psz_name ) != -1 ) { vlc_meta_SetArtURL( p_meta, psz_url ); free( psz_url ); } } } fields.erase(iter); }
static void get_str16(WMA_File *f, String& s) { int len; //, c; // char *q; len = get_le16(f); ByteVector bv = f->ReadBlock(len * 2); // len is number of wide characters (see below) bv.append( ByteVector::fromShort(0) ); // NULL terminator s = String((wchar_t *) bv.data(), String::UTFLE16); // q = buf; // while (len > 0) { // c = get_le16(f); // if ((q - buf) < buf_size - 1) // *q++ = c; // len--; // } // *q = '\0'; }
///--------------------------------------------------------------------------------- /// ///--------------------------------------------------------------------------------- bool LoadBinaryFileToExistingByteVector( const std::string& filePath, ByteVector& existingVectorBuffer ) { FILE* file; fopen_s( &file, filePath.c_str(), "rb" ); if (!file) { // freak out return false; } size_t neededBufferSize = GetFileLength( file ); // Grow/Shrink existingVectorBuffer.resize( neededBufferSize ); fread( existingVectorBuffer.data(), sizeof( unsigned char ), neededBufferSize, file ); fclose( file ); return true; }
size_t IFile::read(ByteVector &vector) { size_t cnt = 0, oldPos, size; char *buffer; if (stream == NULL) return 0; oldPos = ftell(stream); fseek(stream, 0, SEEK_END); size = ftell(stream); fseek(stream, 0, SEEK_SET); vector.resize(size); buffer = vector.data(); while (cnt < size && !feof(stream) && !ferror(stream)) fread(buffer, 1, size - cnt, stream); if (!ferror(stream)) fseek(stream, oldPos, SEEK_SET); return cnt; }
bool TagEditor::save() { if ( fRef ) { bool mustSave = false, result = false; if ( !isChecked() ) clearValues(); File &file = *fRef->file(); Tag &tag = getTag( *fRef, file ); if ( titleE->text() != tag.title().toCString( true ) ) { tag.setTitle( String( titleE->text().toUtf8().data(), String::UTF8 ) ); mustSave = true; } if ( artistE->text() != tag.artist().toCString( true ) ) { tag.setArtist( String( artistE->text().toUtf8().data(), String::UTF8 ) ); mustSave = true; } if ( albumE->text() != tag.album().toCString( true ) ) { tag.setAlbum( String( albumE->text().toUtf8().data(), String::UTF8 ) ); mustSave = true; } if ( commentE->text() != tag.comment().toCString( true ) ) { tag.setComment( String( commentE->text().toUtf8().data(), String::UTF8 ) ); mustSave = true; } if ( genreE->text() != tag.genre().toCString( true ) ) { tag.setGenre( String( genreE->text().toUtf8().data(), String::UTF8 ) ); mustSave = true; } if ( ( uint )yearB->value() != tag.year() ) { tag.setYear( yearB->value() ); mustSave = true; } if ( ( uint )trackB->value() != tag.track() ) { tag.setTrack( trackB->value() ); mustSave = true; } if ( isChecked() && ( pictureModificated || pictureBChecked != pictureB->isChecked() ) ) { const bool hasPicture = pictureB->isChecked() && !picture->isEmpty(); if ( instanceOf( file, MPEG::File ) || instanceOf( file, RIFF::AIFF::File ) ) { ID3v2::Tag *id3v2 = NULL; if ( instanceOf( file, MPEG::File ) ) id3v2 = ( ( MPEG::File & )file ).ID3v2Tag( hasPicture ); else if ( instanceOf( file, RIFF::AIFF::File ) ) id3v2 = ( ( RIFF::AIFF::File & )file ).tag(); if ( id3v2 ) { id3v2->removeFrames( "APIC" ); if ( hasPicture ) { ID3v2::AttachedPictureFrame *pictureFrame = new ID3v2::AttachedPictureFrame; pictureFrame->setType( ID3v2::AttachedPictureFrame::FrontCover ); pictureFrame->setMimeType( pictureMimeType.data() ); pictureFrame->setPicture( *picture ); id3v2->addFrame( pictureFrame ); } mustSave = true; } } else if ( instanceOf( file, FLAC::File ) ) { FLAC::File &flacF = ( FLAC::File & )file; flacF.removePictures(); if ( hasPicture ) { FLAC::Picture *flacPicture = new FLAC::Picture; flacPicture->setMimeType( pictureMimeType.data() ); flacPicture->setType( FLAC::Picture::FrontCover ); flacPicture->setData( *picture ); flacF.addPicture( flacPicture ); } mustSave = true; } else if ( instanceOf( file, MP4::File ) ) { MP4::ItemListMap &itemListMap = ( ( MP4::File & )file ).tag()->itemListMap(); if ( itemListMap.contains( "covr" ) ) itemListMap.erase( "covr" ); if ( hasPicture ) { MP4::CoverArt::Format format = ( MP4::CoverArt::Format )0; if ( pictureMimeType == "image/jpeg" ) format = MP4::CoverArt::JPEG; else if ( pictureMimeType == "image/png" ) format = MP4::CoverArt::PNG; #if TAGLIB18 else if ( pictureMimeType == "image/bmp" ) format = MP4::CoverArt::BMP; else if ( pictureMimeType == "image/gif" ) format = MP4::CoverArt::GIF; #endif if ( format ) { MP4::CoverArtList coverArtList; coverArtList.append( MP4::CoverArt( format, *picture ) ); itemListMap.insert( "covr", coverArtList ); } } mustSave = true; } else if ( isOgg( file ) ) { Ogg::XiphComment *xiphComment = getXiphComment( file ); if ( xiphComment ) { xiphComment->removeField( "METADATA_BLOCK_PICTURE" ); if ( hasPicture ) { FLAC::Picture flacPicture; flacPicture.setMimeType( pictureMimeType.data() ); flacPicture.setType( FLAC::Picture::FrontCover ); flacPicture.setData( *picture ); const ByteVector pict_data = flacPicture.render(); xiphComment->addField( "METADATA_BLOCK_PICTURE", QByteArray::fromRawData( pict_data.data(), pict_data.size() ).toBase64().data() ); } mustSave = true; } } } else if ( !isChecked() ) //Usuwanie wszystkich znanych tagów { mustSave = true; if ( instanceOf( file, MPEG::File ) ) ( ( MPEG::File & )file ).strip(); else if ( instanceOf( file, MPC::File ) ) ( ( MPC::File & )file ).strip(); else if ( instanceOf( file, WavPack::File ) ) ( ( WavPack::File & )file ).strip(); else if ( instanceOf( file, TrueAudio::File ) ) ( ( TrueAudio::File & )file ).strip(); else if ( instanceOf( file, APE::File ) ) ( ( APE::File & )file ).strip(); else if ( instanceOf( file, MP4::File ) ) ( ( MP4::File & )file ).tag()->itemListMap().clear(); else if ( instanceOf( file, ASF::File ) ) ( ( ASF::File & )file ).tag()->attributeListMap().clear(); else if ( isOgg( file ) ) removeXiphComment( getXiphComment( file ) ); else if ( instanceOf( file, FLAC::File ) ) { FLAC::File &flacF = ( FLAC::File & )file; flacF.removePictures(); #if TAGLIB19 if ( flacF.hasXiphComment() ) #endif removeXiphComment( flacF.xiphComment() ); } else if ( instanceOf( file, RIFF::AIFF::File ) ) { ID3v2::Tag *id3v2 = ( ( RIFF::AIFF::File & )file ).tag(); if ( id3v2 ) { ID3v2::FrameList frameList = id3v2->frameList(); for ( ID3v2::FrameList::ConstIterator it = frameList.begin() ; it != frameList.end() ; ++it ) id3v2->removeFrame( *it ); } } #if TAGLIB18 else if ( instanceOf( file, Mod::File ) || instanceOf( file, S3M::File ) || instanceOf( file, IT::File ) || instanceOf( file, XM::File ) ) { Mod::Tag *modTag = NULL; if ( instanceOf( file, Mod::File ) ) modTag = ( ( Mod::File & )file ).tag(); else if ( instanceOf( file, S3M::File ) ) modTag = ( ( S3M::File & )file ).tag(); else if ( instanceOf( file, IT::File ) ) modTag = ( ( IT::File & )file ).tag(); else if ( instanceOf( file, XM::File ) ) modTag = ( ( XM::File & )file ).tag(); if ( modTag ) modTag->setTrackerName( String::null ); } #endif } /* FLAC::File writer BUG workaround - remove ID3 tags */ if ( mustSave && instanceOf( file, FLAC::File ) ) { FLAC::File &flacF = ( FLAC::File & )file; #if TAGLIB19 if ( flacF.hasID3v1Tag() || flacF.hasID3v2Tag() ) #else if ( flacF.ID3v1Tag() || flacF.ID3v2Tag() ) #endif { const FileName fName = fRef->file()->name(); result = fRef->save(); delete fRef; fRef = NULL; if ( result ) result = MPEG::File( fName, false ).save( MPEG::File::NoTags ); mustSave = false; } } #if TAGLIB19 /* No ID3v2 in WAV, only InfoTag */ if ( mustSave && instanceOf( file, RIFF::WAV::File ) ) { RIFF::WAV::File &wavF = ( RIFF::WAV::File & )file; wavF.save( wavF.InfoTag()->isEmpty() ? RIFF::WAV::File::NoTags : RIFF::WAV::File::Info ); mustSave = false; } #endif return mustSave ? fRef->save() : ( fRef ? true : result ); } return false; }
/*! * Attempts to write the block \a data at the current get pointer. If the * file is currently only opened read only -- i.e. readOnly() returns true -- * this attempts to reopen the file in read/write mode. * * \note This should be used instead of using the streaming output operator * for a ByteVector. And even this function is significantly slower than * doing output with a char[]. */ void TagLibVFSStream::writeBlock(const ByteVector &data) { m_file.Write(data.data(), data.size()); }
/** * Read meta information from id3v2 tags * @param tag: the id3v2 tag * @param p_demux; the demux object * @param p_demux_meta: the demuxer meta * @param p_meta: the meta */ static void ReadMetaFromId3v2( ID3v2::Tag* tag, demux_t* p_demux, demux_meta_t* p_demux_meta, vlc_meta_t* p_meta ) { // Get the unique file identifier ID3v2::FrameList list = tag->frameListMap()["UFID"]; ID3v2::FrameList::Iterator iter; for( iter = list.begin(); iter != list.end(); iter++ ) { ID3v2::UniqueFileIdentifierFrame* p_ufid = dynamic_cast<ID3v2::UniqueFileIdentifierFrame*>(*iter); const char *owner = p_ufid->owner().toCString(); if (!strcmp( owner, "http://musicbrainz.org" )) { /* ID3v2 UFID contains up to 64 bytes binary data * but in our case it will be a '\0' * terminated string */ char psz_ufid[64]; int max_size = __MIN( p_ufid->identifier().size(), 63); strncpy( psz_ufid, p_ufid->identifier().data(), max_size ); psz_ufid[max_size] = '\0'; vlc_meta_SetTrackID( p_meta, psz_ufid ); } } // Get the use text list = tag->frameListMap()["TXXX"]; for( iter = list.begin(); iter != list.end(); iter++ ) { ID3v2::UserTextIdentificationFrame* p_txxx = dynamic_cast<ID3v2::UserTextIdentificationFrame*>(*iter); vlc_meta_AddExtra( p_meta, p_txxx->description().toCString( true ), p_txxx->fieldList().toString().toCString( true ) ); } // Get some more informations #define SET( tagName, metaName ) \ list = tag->frameListMap()[tagName]; \ if( !list.isEmpty() ) \ vlc_meta_Set##metaName( p_meta, \ (*list.begin())->toString().toCString( true ) ); SET( "TCOP", Copyright ); SET( "TENC", EncodedBy ); SET( "TLAN", Language ); SET( "TPUB", Publisher ); #undef SET /* Preferred type of image * The 21 types are defined in id3v2 standard: * http://www.id3.org/id3v2.4.0-frames */ static const int pi_cover_score[] = { 0, /* Other */ 5, /* 32x32 PNG image that should be used as the file icon */ 4, /* File icon of a different size or format. */ 20, /* Front cover image of the album. */ 19, /* Back cover image of the album. */ 13, /* Inside leaflet page of the album. */ 18, /* Image from the album itself. */ 17, /* Picture of the lead artist or soloist. */ 16, /* Picture of the artist or performer. */ 14, /* Picture of the conductor. */ 15, /* Picture of the band or orchestra. */ 9, /* Picture of the composer. */ 8, /* Picture of the lyricist or text writer. */ 7, /* Picture of the recording location or studio. */ 10, /* Picture of the artists during recording. */ 11, /* Picture of the artists during performance. */ 6, /* Picture from a movie or video related to the track. */ 1, /* Picture of a large, coloured fish. */ 12, /* Illustration related to the track. */ 3, /* Logo of the band or performer. */ 2 /* Logo of the publisher (record company). */ }; int i_score = -1; // Try now to get embedded art list = tag->frameListMap()[ "APIC" ]; if( list.isEmpty() ) return; TAB_INIT( p_demux_meta->i_attachments, p_demux_meta->attachments ); for( iter = list.begin(); iter != list.end(); iter++ ) { ID3v2::AttachedPictureFrame* p_apic = dynamic_cast<ID3v2::AttachedPictureFrame*>(*iter); input_attachment_t *p_attachment; const char *psz_mime; char *psz_name, *psz_description; // Get the mime and description of the image. // If the description is empty, take the type as a description psz_mime = p_apic->mimeType().toCString( true ); if( p_apic->description().size() > 0 ) psz_description = strdup( p_apic->description().toCString( true ) ); else { if( asprintf( &psz_description, "%i", p_apic->type() ) == -1 ) psz_description = NULL; } if( !psz_description ) continue; psz_name = psz_description; /* some old iTunes version not only sets incorrectly the mime type * or the description of the image, * but also embeds incorrectly the image. * Recent versions seem to behave correctly */ if( !strncmp( psz_mime, "PNG", 3 ) || !strncmp( psz_name, "\xC2\x89PNG", 5 ) ) { msg_Warn( p_demux, "Invalid picture embedded by broken iTunes version" ); free( psz_description ); continue; } const ByteVector picture = p_apic->picture(); const char *p_data = picture.data(); const unsigned i_data = picture.size(); msg_Dbg( p_demux, "Found embedded art: %s (%s) is %u bytes", psz_name, psz_mime, i_data ); p_attachment = vlc_input_attachment_New( psz_name, psz_mime, psz_description, p_data, i_data ); if( p_attachment ) TAB_APPEND_CAST( (input_attachment_t**), p_demux_meta->i_attachments, p_demux_meta->attachments, p_attachment ); free( psz_description ); if( pi_cover_score[p_apic->type()] > i_score ) { i_score = pi_cover_score[p_apic->type()]; char *psz_url; if( asprintf( &psz_url, "attachment://%s", p_attachment->psz_name ) == -1 ) continue; vlc_meta_SetArtURL( p_meta, psz_url ); free( psz_url ); } } }
void File::insert(const ByteVector &data, ulong start, ulong replace) { if(!d->file) return; if(data.size() == replace) { seek(start); writeBlock(data); return; } else if(data.size() < replace) { seek(start); writeBlock(data); removeBlock(start + data.size(), replace - data.size()); return; } // Woohoo! Faster (about 20%) than id3lib at last. I had to get hardcore // and avoid TagLib's high level API for rendering just copying parts of // the file that don't contain tag data. // // Now I'll explain the steps in this ugliness: // First, make sure that we're working with a buffer that is longer than // the *differnce* in the tag sizes. We want to avoid overwriting parts // that aren't yet in memory, so this is necessary. ulong bufferLength = bufferSize(); while(data.size() - replace > bufferLength) bufferLength += bufferSize(); // Set where to start the reading and writing. long readPosition = start + replace; long writePosition = start; ByteVector buffer; ByteVector aboutToOverwrite(static_cast<uint>(bufferLength)); // This is basically a special case of the loop below. Here we're just // doing the same steps as below, but since we aren't using the same buffer // size -- instead we're using the tag size -- this has to be handled as a // special case. We're also using File::writeBlock() just for the tag. // That's a bit slower than using char *'s so, we're only doing it here. seek(readPosition); int bytesRead = fread(aboutToOverwrite.data(), sizeof(char), bufferLength, d->file); readPosition += bufferLength; seek(writePosition); writeBlock(data); writePosition += data.size(); buffer = aboutToOverwrite; // In case we've already reached the end of file... buffer.resize(bytesRead); // Ok, here's the main loop. We want to loop until the read fails, which // means that we hit the end of the file. while(!buffer.isEmpty()) { // Seek to the current read position and read the data that we're about // to overwrite. Appropriately increment the readPosition. seek(readPosition); bytesRead = fread(aboutToOverwrite.data(), sizeof(char), bufferLength, d->file); aboutToOverwrite.resize(bytesRead); readPosition += bufferLength; // Check to see if we just read the last block. We need to call clear() // if we did so that the last write succeeds. if(ulong(bytesRead) < bufferLength) clear(); // Seek to the write position and write our buffer. Increment the // writePosition. seek(writePosition); fwrite(buffer.data(), sizeof(char), buffer.size(), d->file); writePosition += buffer.size(); // Make the current buffer the data that we read in the beginning. buffer = aboutToOverwrite; // Again, we need this for the last write. We don't want to write garbage // at the end of our file, so we need to set the buffer size to the amount // that we actually read. bufferLength = bytesRead; } }
int main(int argc, char** argv) { const uint64_t N_TIMESTAMPS = 1000; const uint64_t N_PARAMS = 100; UncompressedChunk header; std::cout << "Testing timestamp sequence" << std::endl; int c = 100; std::vector<aku_ParamId> ids; for (uint64_t id = 0; id < N_PARAMS; id++) { ids.push_back(id); } RandomWalk rwalk(10.0, 0.0, 0.01, N_PARAMS); for (uint64_t id = 0; id < N_PARAMS; id++) { for (uint64_t ts = 0; ts < N_TIMESTAMPS; ts++) { header.paramids.push_back(ids[id]); int k = rand() % 2; if (k) { c++; } else if (c > 0) { c--; } header.timestamps.push_back((ts + c) << 8); header.values.push_back(rwalk.generate(0)); } } ByteVector out; out.resize(N_PARAMS*N_TIMESTAMPS*24); const size_t UNCOMPRESSED_SIZE = header.paramids.size()*8 // Didn't count lengths and offsets + header.timestamps.size()*8 // because because this arrays contains + header.values.size()*8; // no information and should be compressed // to a few bytes struct Writer : ChunkWriter { ByteVector *out; Writer(ByteVector *out) : out(out) {} virtual aku_MemRange allocate() { aku_MemRange range = { out->data(), static_cast<uint32_t>(out->size()) }; return range; } //! Commit changes virtual aku_Status commit(size_t bytes_written) { out->resize(bytes_written); return AKU_SUCCESS; } }; Writer writer(&out); aku_Timestamp tsbegin, tsend; uint32_t n; auto status = CompressionUtil::encode_chunk(&n, &tsbegin, &tsend, &writer, header); if (status != AKU_SUCCESS) { std::cout << "Encoding error" << std::endl; return 1; } // Compress using zlib // Ids copy (zlib need all input data to be aligned because it uses SSE2 internally) Bytef* pgz_ids = (Bytef*)aligned_alloc(64, header.paramids.size()*8); memcpy(pgz_ids, header.paramids.data(), header.paramids.size()*8); // Timestamps copy Bytef* pgz_ts = (Bytef*)aligned_alloc(64, header.timestamps.size()*8); memcpy(pgz_ts, header.timestamps.data(), header.timestamps.size()*8); // Values copy Bytef* pgz_val = (Bytef*)aligned_alloc(64, header.values.size()*8); memcpy(pgz_val, header.values.data(), header.values.size()*8); const auto gz_max_size = N_PARAMS*N_TIMESTAMPS*24; Bytef* pgzout = (Bytef*)aligned_alloc(64, gz_max_size); uLongf gzoutlen = gz_max_size; size_t total_gz_size = 0, id_gz_size = 0, ts_gz_size = 0, float_gz_size = 0; // compress param ids auto zstatus = compress(pgzout, &gzoutlen, pgz_ids, header.paramids.size()*8); if (zstatus != Z_OK) { std::cout << "GZip error" << std::endl; exit(zstatus); } total_gz_size += gzoutlen; id_gz_size = gzoutlen; gzoutlen = gz_max_size; // compress timestamps zstatus = compress(pgzout, &gzoutlen, pgz_ts, header.timestamps.size()*8); if (zstatus != Z_OK) { std::cout << "GZip error" << std::endl; exit(zstatus); } total_gz_size += gzoutlen; ts_gz_size = gzoutlen; gzoutlen = gz_max_size; // compress floats zstatus = compress(pgzout, &gzoutlen, pgz_val, header.values.size()*8); if (zstatus != Z_OK) { std::cout << "GZip error" << std::endl; exit(zstatus); } total_gz_size += gzoutlen; float_gz_size = gzoutlen; const float GZ_BPE = float(total_gz_size)/header.paramids.size(); const float GZ_RATIO = float(UNCOMPRESSED_SIZE)/float(total_gz_size); const size_t COMPRESSED_SIZE = out.size(); const float BYTES_PER_EL = float(COMPRESSED_SIZE)/header.paramids.size(); const float COMPRESSION_RATIO = float(UNCOMPRESSED_SIZE)/COMPRESSED_SIZE; std::cout << "Uncompressed: " << UNCOMPRESSED_SIZE << std::endl << " compressed: " << COMPRESSED_SIZE << std::endl << " elements: " << header.paramids.size() << std::endl << " bytes/elem: " << BYTES_PER_EL << std::endl << " ratio: " << COMPRESSION_RATIO << std::endl ; std::cout << "Gzip stats: " << std::endl << "bytes/elem: " << GZ_BPE << std::endl << " ratio: " << GZ_RATIO << std::endl << " id bytes: " << id_gz_size << std::endl << " ts bytes: " << ts_gz_size << std::endl << " val bytes: " << float_gz_size << std::endl; // Try to decompress UncompressedChunk decomp; const unsigned char* pbegin = out.data(); const unsigned char* pend = pbegin + out.size(); CompressionUtil::decode_chunk(&decomp, pbegin, pend, header.timestamps.size()); bool first_error = true; for (auto i = 0u; i < header.timestamps.size(); i++) { if (header.timestamps.at(i) != decomp.timestamps.at(i) && first_error) { std::cout << "Error, bad timestamp at " << i << std::endl; first_error = false; } if (header.paramids.at(i) != decomp.paramids.at(i) && first_error) { std::cout << "Error, bad paramid at " << i << std::endl; first_error = false; } double origvalue = header.values.at(i); double decvalue = decomp.values.at(i); if (origvalue != decvalue && first_error) { std::cout << "Error, bad value at " << i << std::endl; std::cout << "Expected: " << origvalue << std::endl; std::cout << "Actual: " << decvalue << std::endl; first_error = false; } } if (argc == 2 && std::string(argv[1]) == "benchmark") { // Bench compression process const int NRUNS = 1000; PerfTimer tm; aku_Status tstatus; volatile uint32_t vn; ByteVector vec; for (int i = 0; i < NRUNS; i++) { vec.resize(N_PARAMS*N_TIMESTAMPS*24); Writer w(&vec); aku_Timestamp ts; uint32_t n; tstatus = CompressionUtil::encode_chunk(&n, &ts, &ts, &w, header); if (tstatus != AKU_SUCCESS) { std::cout << "Encoding error" << std::endl; return 1; } vn = n; } double elapsed = tm.elapsed(); std::cout << "Elapsed (akumuli): " << elapsed << " " << vn << std::endl; tm.restart(); for (int i = 0; i < NRUNS; i++) { uLongf offset = 0; // compress param ids auto zstatus = compress(pgzout, &gzoutlen, pgz_ids, header.paramids.size()*8); if (zstatus != Z_OK) { std::cout << "GZip error" << std::endl; exit(zstatus); } offset += gzoutlen; gzoutlen = gz_max_size - offset; // compress timestamps zstatus = compress(pgzout + offset, &gzoutlen, pgz_ts, header.timestamps.size()*8); if (zstatus != Z_OK) { std::cout << "GZip error" << std::endl; exit(zstatus); } offset += gzoutlen; gzoutlen = gz_max_size - offset; // compress floats zstatus = compress(pgzout + offset, &gzoutlen, pgz_val, header.values.size()*8); if (zstatus != Z_OK) { std::cout << "GZip error" << std::endl; exit(zstatus); } } elapsed = tm.elapsed(); std::cout << "Elapsed (zlib): " << elapsed << " " << vn << std::endl; } }
int main() { const uint64_t N_TIMESTAMPS = 100; const uint64_t N_PARAMS = 100; UncompressedChunk header; std::cout << "Testing timestamp sequence" << std::endl; int c = 100; std::vector<aku_ParamId> ids; for (uint64_t id = 0; id < N_PARAMS; id++) { ids.push_back(id); } //std::random_shuffle(ids.begin(), ids.end()); for (uint64_t id = 0; id < N_PARAMS; id++) { for (uint64_t ts = 0; ts < N_TIMESTAMPS; ts++) { header.paramids.push_back(ids[id]); int k = rand() % 2; if (k) { c++; } else if (c > 0) { c--; } header.timestamps.push_back(ts + c); ChunkValue cvalue; cvalue.type = ChunkValue::FLOAT; cvalue.value.floatval = id + ts; header.values.push_back(cvalue); } } ByteVector out; out.resize(N_PARAMS*N_TIMESTAMPS*24); const size_t UNCOMPRESSED_SIZE = header.paramids.size()*8 // Didn't count lengths and offsets + header.timestamps.size()*8 // because because this arrays contains + header.values.size()*8; // no information and should be compressed // to a few bytes struct Writer : ChunkWriter { ByteVector *out; Writer(ByteVector *out) : out(out) {} virtual aku_MemRange allocate() { aku_MemRange range = { out->data(), static_cast<uint32_t>(out->size()) }; return range; } //! Commit changes virtual aku_Status commit(size_t bytes_written) { return AKU_SUCCESS; } } writer(&out); aku_Timestamp tsbegin, tsend; uint32_t n; auto status = CompressionUtil::encode_chunk(&n, &tsbegin, &tsend, &writer, header); if (status != AKU_SUCCESS) { std::cout << "Encoding error" << std::endl; return 1; } const size_t COMPRESSED_SIZE = out.size(); std::cout << "Uncompressed: " << UNCOMPRESSED_SIZE << ", compressed: " << COMPRESSED_SIZE << std::endl; // Try to decompress UncompressedChunk decomp; const unsigned char* pbegin = out.data(); const unsigned char* pend = pbegin + out.size(); CompressionUtil::decode_chunk(&decomp, pbegin, pend, header.timestamps.size()); for (auto i = 0u; i < header.timestamps.size(); i++) { if (header.timestamps.at(i) != decomp.timestamps.at(i)) { std::cout << "Error, bad timestamp at " << i << std::endl; } if (header.paramids.at(i) != decomp.paramids.at(i)) { std::cout << "Error, bad paramid at " << i << std::endl; } ChunkValue origvalue = header.values.at(i); ChunkValue decvalue = decomp.values.at(i); if (origvalue.type != decvalue.type) { std::cout << "Error, bad type at " << i << std::endl; } else { if (origvalue.type == ChunkValue::FLOAT) { if (origvalue.value.floatval != decvalue.value.floatval) { std::cout << "Error, bad value at " << i << std::endl; } } else { if (origvalue.value.blobval.length != decvalue.value.blobval.length) { std::cout << "Error, bad length at " << i << std::endl; } if (origvalue.value.blobval.offset != decvalue.value.blobval.offset) { std::cout << "Error, bad offset at " << i << std::endl; } } } } }