static int cbm_write(struct exfat_dev* dev) { uint32_t allocated_clusters = DIV_ROUND_UP(cbm.get_size(), get_cluster_size()) + DIV_ROUND_UP(uct.get_size(), get_cluster_size()) + DIV_ROUND_UP(rootdir.get_size(), get_cluster_size()); size_t bitmap_size = DIV_ROUND_UP(allocated_clusters, CHAR_BIT); bitmap_t* bitmap = malloc(BMAP_SIZE(bitmap_size)); size_t i; if (bitmap == NULL) { exfat_error("failed to allocate bitmap of %zu bytes", BMAP_SIZE(bitmap_size)); return 1; } memset(bitmap, 0, BMAP_SIZE(bitmap_size)); for (i = 0; i < bitmap_size * CHAR_BIT; i++) if (i < allocated_clusters) BMAP_SET(bitmap, i); if (exfat_write(dev, bitmap, bitmap_size) < 0) { free(bitmap); exfat_error("failed to write bitmap of %zu bytes", bitmap_size); return 1; } free(bitmap); return 0; }
static void init_sb(struct exfat_super_block* sb) { uint32_t clusters_max; uint32_t fat_sectors; clusters_max = get_volume_size() / get_cluster_size(); fat_sectors = DIV_ROUND_UP((loff_t) clusters_max * sizeof(cluster_t), get_sector_size()); memset(sb, 0, sizeof(struct exfat_super_block)); sb->jump[0] = 0xeb; sb->jump[1] = 0x76; sb->jump[2] = 0x90; memcpy(sb->oem_name, "EXFAT ", sizeof(sb->oem_name)); sb->sector_start = cpu_to_le64(get_first_sector()); sb->sector_count = cpu_to_le64(get_volume_size() / get_sector_size()); sb->fat_sector_start = cpu_to_le32( fat.get_alignment() / get_sector_size()); sb->fat_sector_count = cpu_to_le32(ROUND_UP( le32_to_cpu(sb->fat_sector_start) + fat_sectors, 1 << get_spc_bits()) - le32_to_cpu(sb->fat_sector_start)); sb->cluster_sector_start = cpu_to_le32( get_position(&cbm) / get_sector_size()); sb->cluster_count = cpu_to_le32(clusters_max - ((le32_to_cpu(sb->fat_sector_start) + le32_to_cpu(sb->fat_sector_count)) >> get_spc_bits())); sb->rootdir_cluster = cpu_to_le32( (get_position(&rootdir) - get_position(&cbm)) / get_cluster_size() + EXFAT_FIRST_DATA_CLUSTER); sb->volume_serial = cpu_to_le32(get_volume_serial()); sb->version.major = 1; sb->version.minor = 0; sb->volume_state = cpu_to_le16(0); sb->sector_bits = get_sector_bits(); sb->spc_bits = get_spc_bits(); sb->fat_count = 1; sb->drive_no = 0x80; sb->allocated_percent = 0; sb->boot_signature = cpu_to_le16(0xaa55); }
static cluster_t fat_write_entries(struct exfat_dev* dev, cluster_t cluster, uint64_t length) { cluster_t end = cluster + DIV_ROUND_UP(length, get_cluster_size()); while (cluster < end - 1) { cluster = fat_write_entry(dev, cluster, cluster + 1); if (cluster == 0) return 0; } return fat_write_entry(dev, cluster, EXFAT_CLUSTER_END); }
TEST_F(RestartTest, happy_path) { const size_t vsize = 1 << 20; const vfs::FrontendPath fname1(make_volume_name("/volume1")); const vd::VolumeId vname1(create_file(fname1, vsize)); const size_t csize = get_cluster_size(vfs::ObjectId(vname1.str())); const vfs::FrontendPath fname2(make_volume_name("/volume2")); const vd::VolumeId vname2(create_file(fname2, vsize)); const vfs::FrontendPath fname3("/some.file.that.is.not.a.volume"); create_file(fname3, vsize); const std::string pattern1("Herzog"); const size_t wsize = 2 * csize; const off_t off = csize - 1; write_to_file(fname1, pattern1, wsize, off); const std::string pattern2("Springtime Epigram"); write_to_file(fname2, pattern2, wsize, off); const std::string pattern3("Ted"); write_to_file(fname3, pattern3, wsize, off); { LOCKVD(); EXPECT_NO_THROW(api::getVolumePointer(vname1)); EXPECT_NO_THROW(api::getVolumePointer(vname2)); } stop_fs(); start_fs(); { LOCKVD(); EXPECT_NO_THROW(api::getVolumePointer(vname1)); EXPECT_NO_THROW(api::getVolumePointer(vname2)); } check_file(fname1, pattern1, wsize, off); check_file(fname2, pattern2, wsize, off); check_file(fname3, pattern3, wsize, off); }
static off_t uct_alignment(void) { return get_cluster_size(); }
static void do_standby_clone(void) { PGconn *conn; PGresult *res; char sqlquery[QUERY_STR_LEN]; int r = 0; int i; bool pg_dir = false; char master_data_directory[MAXLEN]; char master_config_file[MAXLEN]; char master_hba_file[MAXLEN]; char master_ident_file[MAXLEN]; char master_control_file[MAXLEN]; char local_control_file[MAXLEN]; const char *first_wal_segment = NULL; const char *last_wal_segment = NULL; char master_version[MAXVERSIONSTR]; /* if dest_dir hasn't been provided, initialize to current directory */ if (dest_dir == NULL) { dest_dir = malloc(5); strcpy(dest_dir, "."); } /* Check this directory could be used as a PGDATA dir */ switch (check_dir(dest_dir)) { case 0: /* dest_dir not there, must create it */ if (verbose) printf(_("creating directory %s ... "), dest_dir); fflush(stdout); if (!create_directory(dest_dir)) { fprintf(stderr, _("%s: couldn't create directory %s ... "), progname, dest_dir); return; } break; case 1: /* Present but empty, fix permissions and use it */ if (verbose) printf(_("fixing permissions on existing directory %s ... "), dest_dir); fflush(stdout); if (!set_directory_permissions(dest_dir)) { fprintf(stderr, _("%s: could not change permissions of directory \"%s\": %s\n"), progname, dest_dir, strerror(errno)); return; } break; case 2: /* Present and not empty */ fprintf(stderr, _("%s: directory \"%s\" exists but is not empty\n"), progname, dest_dir); pg_dir = is_pg_dir(dest_dir); if (pg_dir && !force) { fprintf(stderr, _("\nThis looks like a PostgreSQL directroy.\n" "If you are sure you want to clone here, " "please check there is no PostgreSQL server " "running and use the --force option\n")); return; } else if (pg_dir && force) { /* Let it continue */ break; } else return; default: /* Trouble accessing directory */ fprintf(stderr, _("%s: could not access directory \"%s\": %s\n"), progname, dest_dir, strerror(errno)); } /* Connection parameters for master only */ keywords[0] = "host"; values[0] = host; keywords[1] = "port"; values[1] = masterport; /* We need to connect to check configuration and start a backup */ conn = PQconnectdbParams(keywords, values, true); if (!conn) { fprintf(stderr, _("%s: could not connect to master\n"), progname); return; } /* primary should be v9 or better */ pg_version(conn, master_version); if (strcmp(master_version, "") == 0) { PQfinish(conn); fprintf(stderr, _("%s needs master to be PostgreSQL 9.0 or better\n"), progname); return; } /* Check we are cloning a primary node */ if (is_standby(conn)) { PQfinish(conn); fprintf(stderr, "\nThe command should clone a primary node\n"); return; } /* And check if it is well configured */ if (!guc_setted(conn, "wal_level", "=", "hot_standby")) { PQfinish(conn); fprintf(stderr, _("%s needs parameter 'wal_level' to be set to 'hot_standby'\n"), progname); return; } if (!guc_setted(conn, "wal_keep_segments", ">=", wal_keep_segments)) { PQfinish(conn); fprintf(stderr, _("%s needs parameter 'wal_keep_segments' to be set to %s or greater\n"), wal_keep_segments, progname); return; } if (!guc_setted(conn, "archive_mode", "=", "on")) { PQfinish(conn); fprintf(stderr, _("%s needs parameter 'archive_mode' to be set to 'on'\n"), progname); return; } if (verbose) printf(_("Succesfully connected to primary. Current installation size is %s\n"), get_cluster_size(conn)); /* Check if the tablespace locations exists and that we can write to them */ sprintf(sqlquery, "select spclocation from pg_tablespace where spcname not in ('pg_default', 'pg_global')"); res = PQexec(conn, sqlquery); if (PQresultStatus(res) != PGRES_TUPLES_OK) { fprintf(stderr, "Can't get info about tablespaces: %s\n", PQerrorMessage(conn)); PQclear(res); PQfinish(conn); return; } for (i = 0; i < PQntuples(res); i++) { char *tblspc_dir = NULL; strcpy(tblspc_dir, PQgetvalue(res, i, 0)); /* Check this directory could be used as a PGDATA dir */ switch (check_dir(tblspc_dir)) { case 0: /* tblspc_dir not there, must create it */ if (verbose) printf(_("creating directory \"%s\"... "), tblspc_dir); fflush(stdout); if (!create_directory(tblspc_dir)) { fprintf(stderr, _("%s: couldn't create directory \"%s\"... "), progname, tblspc_dir); PQclear(res); PQfinish(conn); return; } break; case 1: /* Present but empty, fix permissions and use it */ if (verbose) printf(_("fixing permissions on existing directory \"%s\"... "), tblspc_dir); fflush(stdout); if (!set_directory_permissions(tblspc_dir)) { fprintf(stderr, _("%s: could not change permissions of directory \"%s\": %s\n"), progname, tblspc_dir, strerror(errno)); PQclear(res); PQfinish(conn); return; } break; case 2: /* Present and not empty */ if (!force) { fprintf(stderr, _("%s: directory \"%s\" exists but is not empty\n"), progname, tblspc_dir); PQclear(res); PQfinish(conn); return; } default: /* Trouble accessing directory */ fprintf(stderr, _("%s: could not access directory \"%s\": %s\n"), progname, tblspc_dir, strerror(errno)); PQclear(res); PQfinish(conn); return; } } fprintf(stderr, "Starting backup...\n"); /* Get the data directory full path and the configuration files location */ sprintf(sqlquery, "SELECT name, setting " " FROM pg_settings " " WHERE name IN ('data_directory', 'config_file', 'hba_file', 'ident_file')"); res = PQexec(conn, sqlquery); if (PQresultStatus(res) != PGRES_TUPLES_OK) { fprintf(stderr, "Can't get info about data directory and configuration files: %s\n", PQerrorMessage(conn)); PQclear(res); PQfinish(conn); return; } for (i = 0; i < PQntuples(res); i++) { if (strcmp(PQgetvalue(res, i, 0), "data_directory") == 0) strcpy(master_data_directory, PQgetvalue(res, i, 1)); else if (strcmp(PQgetvalue(res, i, 0), "config_file") == 0) strcpy(master_config_file, PQgetvalue(res, i, 1)); else if (strcmp(PQgetvalue(res, i, 0), "hba_file") == 0) strcpy(master_hba_file, PQgetvalue(res, i, 1)); else if (strcmp(PQgetvalue(res, i, 0), "ident_file") == 0) strcpy(master_ident_file, PQgetvalue(res, i, 1)); else fprintf(stderr, _("uknown parameter: %s"), PQgetvalue(res, i, 0)); } PQclear(res); /* * inform the master we will start a backup and get the first XLog filename * so we can say to the user we need those files */ sprintf(sqlquery, "SELECT pg_xlogfile_name(pg_start_backup('repmgr_standby_clone_%ld'))", time(NULL)); res = PQexec(conn, sqlquery); if (PQresultStatus(res) != PGRES_TUPLES_OK) { fprintf(stderr, "Can't start backup: %s\n", PQerrorMessage(conn)); PQclear(res); PQfinish(conn); return; } first_wal_segment = PQgetvalue(res, 0, 0); PQclear(res); /* * 1) first move global/pg_control * * 2) then move data_directory ommiting the files we have already moved and pg_xlog * content * * 3) finally We need to backup configuration files (that could be on other directories, debian * like systems likes to do that), so look at config_file, hba_file and ident_file but we * can omit external_pid_file ;) * * On error we need to return but before that execute pg_stop_backup() */ /* need to create the global sub directory */ sprintf(master_control_file, "%s/global/pg_control", master_data_directory); sprintf(local_control_file, "%s/global", dest_dir); if (!create_directory(local_control_file)) { fprintf(stderr, _("%s: couldn't create directory %s ... "), progname, dest_dir); goto stop_backup; } r = copy_remote_files(host, remote_user, master_control_file, local_control_file, false); if (r != 0) goto stop_backup; r = copy_remote_files(host, remote_user, master_data_directory, dest_dir, true); if (r != 0) goto stop_backup; /* * Copy tablespace locations, i'm doing this separately because i couldn't find and appropiate * rsync option but besides we could someday make all these rsync happen concurrently */ sprintf(sqlquery, "select spclocation from pg_tablespace where spcname not in ('pg_default', 'pg_global')"); res = PQexec(conn, sqlquery); if (PQresultStatus(res) != PGRES_TUPLES_OK) { fprintf(stderr, "Can't get info about tablespaces: %s\n", PQerrorMessage(conn)); PQclear(res); goto stop_backup; } for (i = 0; i < PQntuples(res); i++) { r = copy_remote_files(host, remote_user, PQgetvalue(res, i, 0), PQgetvalue(res, i, 0), true); if (r != 0) goto stop_backup; } r = copy_remote_files(host, remote_user, master_config_file, dest_dir, false); if (r != 0) goto stop_backup; r = copy_remote_files(host, remote_user, master_hba_file, dest_dir, false); if (r != 0) goto stop_backup; r = copy_remote_files(host, remote_user, master_ident_file, dest_dir, false); if (r != 0) goto stop_backup; stop_backup: /* inform the master that we have finished the backup */ conn = PQconnectdbParams(keywords, values, true); if (!conn) { fprintf(stderr, _("%s: could not connect to master\n"), progname); return; } fprintf(stderr, "Finishing backup...\n"); sprintf(sqlquery, "SELECT pg_xlogfile_name(pg_stop_backup())"); res = PQexec(conn, sqlquery); if (PQresultStatus(res) != PGRES_TUPLES_OK) { fprintf(stderr, "Can't stop backup: %s\n", PQerrorMessage(conn)); PQclear(res); PQfinish(conn); return; } last_wal_segment = PQgetvalue(res, 0, 0); PQclear(res); PQfinish(conn); /* Now, if the rsync failed then exit */ if (r != 0) return; if (verbose) printf(_("%s requires primary to keep WAL files %s until at least %s\n"), progname, first_wal_segment, last_wal_segment); /* we need to create the pg_xlog sub directory too, i'm reusing a variable here */ sprintf(local_control_file, "%s/pg_xlog", dest_dir); if (!create_directory(local_control_file)) { fprintf(stderr, _("%s: couldn't create directory %s, you will need to do it manually...\n"), progname, dest_dir); } /* Finally, write the recovery.conf file */ create_recovery_file(dest_dir); /* We don't start the service because we still may want to move the directory */ return; }
static off_t fat_size(void) { return get_volume_size() / get_cluster_size() * sizeof(cluster_t); }
static off_t cbm_size(void) { return DIV_ROUND_UP( (get_volume_size() - get_position(&cbm)) / get_cluster_size(), CHAR_BIT); }
static off64_t cbm_alignment(void) { return get_cluster_size(); }
void Connection::recvrequestINT(char *cmd, char *local, char *remote, const char *mode) { int oldtype = 0, is_retr; FHandle fout; SOCKET din = INVALID_SOCKET; int ocode, oecode; BOOL oldBrk = FtpSetBreakable(this, -1); FTPCurrentStates oState = CurrentState; FTNNotify ni; if(type == TYPE_A) restart_point = 0; ni.Upload = FALSE; ni.Starting = TRUE; ni.Success = TRUE; ni.RestartPoint = restart_point; ni.Port = ntohs(portnum); ni.Password[0] = 0; //StrCpy( ni.Password, UserPassword, ARRAYSIZE(ni.Password)); StrCpy(ni.User, UserName, ARRAYSIZE(ni.User)); StrCpy(ni.HostName, hostname, ARRAYSIZE(ni.HostName)); StrCpy(ni.LocalFile, local, ARRAYSIZE(ni.LocalFile)); StrCpy(ni.RemoteFile, remote, ARRAYSIZE(ni.RemoteFile)); if(local[0] == '-' && local[1] == 0) { ; } else { fout.Handle = Fopen(local, mode, Opt.SetHiddenOnAbort ? FILE_ATTRIBUTE_HIDDEN : FILE_ATTRIBUTE_NORMAL); if(!fout.Handle) { ErrorCode = GetLastError(); SysError = TRUE; Log(("!Fopen [%s] %s",mode,__WINError())); if(!ConnectMessage(MErrorOpenFile,local,-MRetry)) ErrorCode = ERROR_CANCELLED; //goto abort; return; } Log(("recv file [%d] \"%s\"=%p",Host.IOBuffSize,local,fout.Handle)); if(restart_point != -1) { if(!Fmove(fout.Handle,restart_point)) { ErrorCode = GetLastError(); SysError = TRUE; if(!ConnectMessage(MErrorPosition,local,-MRetry)) ErrorCode = ERROR_CANCELLED; return; } } TrafficInfo->Resume(restart_point == -1 ? 0 : restart_point); } is_retr = StrCmp(cmd,Opt.cmdRetr) == 0; if(proxy && is_retr) { proxtrans(cmd, local, remote); return; } if(!initconn()) { Log(("!initconn")); return; } if(!is_retr) { if(type != TYPE_A) { oldtype = type; setascii(); } } else if(restart_point) { if(!ResumeSupport) { AddCmdLine(FMSG(MResumeRestart)); restart_point = 0; } else if(restart_point != -1) { if(command("%s %I64u",Opt.cmdRest,restart_point) != RPL_CONTINUE) { Log(("!restart SIZE")); return; } } } if(Host.PassiveMode) { din = dataconn(); if(din == INVALID_SOCKET) { Log(("!dataconn: PASV ent")); goto abort; } } if(remote) { if(command("%s %s", cmd, remote) != RPL_PRELIM) { if(oldtype) SetType(oldtype); Log(("!command [%s]",cmd)); fout.Close(); if(Fsize(local)) DeleteFile(local); return; } } else if(command("%s", cmd) != RPL_PRELIM) { if(oldtype) SetType(oldtype); return; } if(!Host.PassiveMode) { din = dataconn(); if(din == INVALID_SOCKET) { Log(("!dataconn: PASV ret")); goto abort; } } /**/ switch(type) { case TYPE_A: case TYPE_I: case TYPE_L: { FtpSetBreakable(this, FALSE); CurrentState = fcsProcessFile; if(fout.Handle && PluginAvailable(PLUGIN_NOTIFY)) FTPNotify().Notify(&ni); DWORD b,e,bw; __int64 totalValue; int b_done; DWORD ind; int b_ost = Host.IOBuffSize, wsz = get_cluster_size(local)*2; if(!wsz || (wsz > b_ost && (wsz /= 2) > b_ost)) wsz = 512; ind = Min(1024*1024, Max(4*wsz, 256*1024)); // 256K - 1M setsockopt(din, SOL_SOCKET, SO_RCVBUF, (char*)&ind, sizeof(ind)); b_done = ind = 0; totalValue = 0; bool unalign = false; GET_TIME(b); bw = b; while(true) { int c; if(wsz != 512 && b_done >= wsz) // pseudo ansync io { DWORD off = 0, rdy = 0, ost = b_done % wsz, top = b_done - ost; while(ioctlsocket(din, FIONREAD, &rdy) && !rdy) { if(Fwrite(fout.Handle,IOBuff+off,wsz) != wsz) goto write_error; if((off += wsz) >= top) break; } if(off) { b_done -= off; if(b_done) memmove(IOBuff, IOBuff+off, b_done); b_ost = Host.IOBuffSize - b_done; } } //Recv c = nb_recv(&din, IOBuff+b_done, b_ost, 0); if(c <= 0) { if(b_done && Fwrite(fout.Handle,IOBuff,b_done) != b_done) goto write_error; if(c < 0) { Log(("gf(%d,%s)=%I64u: !read buff",code,GetSocketErrorSTR(),totalValue)); code = RPL_TRANSFERERROR; goto NormExit; } Log(("gf(%d,%s)=%I64u: read zero",code,GetSocketErrorSTR(),totalValue)); break; } totalValue += c; GET_TIME(e); if(!fout.Handle) { //Add readed to buffer Log(("AddOutput: +%d bytes", c)); AddOutput((BYTE*)IOBuff,c); } else //Write to file { b_done += c; b_ost -= c; if(b_ost < wsz || CMP_TIME(e,bw) >= 3.0) { DWORD ost = 0; if(wsz == 512 || b_done <= wsz) // timeout or very small buffer { if(Fwrite(fout.Handle,IOBuff,b_done) != b_done) goto write_error; if(b_done < wsz) unalign = true; // flag of timeout witing (optimize) } else { // scatter-gatter for RAID in win32 is very bad on large buffer // and when work without RAID synchronous write speed is independ // if buffer size is >= 2*cluster size int off = 0; if(unalign) // was 'timeouted unaligned write' { unalign = false; off = (DWORD)(totalValue % wsz); if(off) { if(Fwrite(fout.Handle,IOBuff,off) != off) goto write_error; b_done -= off; if(b_done < wsz) { memmove(IOBuff, IOBuff+off, b_done); goto skip_sg; } } } ost = b_done % wsz; b_done -= ost; do if(Fwrite(fout.Handle,IOBuff+off,wsz) != wsz) goto write_error; while((off += wsz) < b_done); if(ost) memmove(IOBuff, IOBuff+off, ost); } b_done = ost; skip_sg: b_ost = Host.IOBuffSize - b_done; GET_TIME(e); bw = e; } } ind += c; if(CMP_TIME(e,b) >= 0.5) { b = e; c = ind; ind = 0; //Call user CB if(IOCallback) { if(!TrafficInfo->Callback(c)) { Log(("gf: canceled by CB")); do_cancel: ErrorCode = ERROR_CANCELLED; if(b_done && Fwrite(fout.Handle,IOBuff,b_done) != b_done) { write_error: SysError = TRUE; ErrorCode = GetLastError(); if(ErrorCode == ERROR_SUCCESS) ErrorCode = ERROR_WRITE_FAULT; // for non equal counter Log(("!write local")); } goto abort; } } else //Show Quite progressing if(Opt.ShowIdle && !remote) { char digit[ 20 ]; String str; str.printf("%s%s ", FP_GetMsg(MReaded), FCps(digit,(double)totalValue)); SetLastError(ERROR_SUCCESS); IdleMessage(str.c_str(),Opt.ProcessColor); if(CheckForEsc(FALSE)) goto do_cancel; } } } if(IOCallback) TrafficInfo->Callback(0); break; } } NormExit: FtpSetBreakable(this, oldBrk); ocode = code; oecode = ErrorCode; CurrentState = oState; scClose(data_peer,-1); if(getreply(FALSE) == RPL_ERROR || oldtype && !SetType(oldtype)) { lostpeer(); } else { code = ocode; ErrorCode = oecode; } if(fout.Handle && PluginAvailable(PLUGIN_NOTIFY)) { ni.Starting = FALSE; ni.Success = TRUE; FTPNotify().Notify(&ni); } return; abort: FtpSetBreakable(this, oldBrk); if(!cpend) { Log(("!!!cpend")); } ocode = code; oecode = ErrorCode; CurrentState = oState; if(!SendAbort(din) || (oldtype && !SetType(oldtype))) lostpeer(); else { code = ocode; ErrorCode = oecode; } scClose(data_peer,-1); if(fout.Handle && PluginAvailable(PLUGIN_NOTIFY)) { ni.Starting = FALSE; ni.Success = FALSE; FTPNotify().Notify(&ni); } return; }
static fbx_off_t cbm_alignment(void) { return get_cluster_size(); }