int test_init(struct harness_t *harness_p)
{
#if defined(ARCH_LINUX)
/* Create an empty sd card file. */
system("../../create_sdcard_linux.sh");
file_p = fopen("sdcard", "r+b");
BTASSERT(fat16_init(&fs,
linux_read_block,
linux_write_block,
file_p,
0) == 0);
#else
BTASSERT(spi_init(&spi,
&spi_device[0],
&pin_d6_dev,
SPI_MODE_MASTER,
SPI_SPEED_500KBPS,
0,
0) == 0);
BTASSERT(sd_init(&sd, &spi) == 0);
BTASSERT(sd_start(&sd) == 0);
BTASSERT(fat16_init(&fs,
(fat16_read_t)sd_read_block,
(fat16_write_t)sd_write_block,
&sd,
0) == 0);
#endif
return (0);
}
static int storage_init(fat16_read_t *read_p,
fat16_write_t *write_p,
void **arg_pp)
{
std_printf(FSTR("SD storage.\r\n"));
std_printf(FSTR("spi bitrate = %lu kbps\r\n"),
2 * 16 * SAMPLES_PER_SOCOND / 1024);
/* Initialize SPI for the SD card. */
spi_init(&spi,
&spi_device[0],
&pin_d53_dev,
SPI_MODE_MASTER,
SPI_SPEED_2MBPS,
0,
1);
sd_init(&sd, &spi);
if (sd_start(&sd) != 0) {
return (-1);
}
std_printf(FSTR("sd card started\r\n"));
*read_p = (fat16_read_t)sd_read_block;
*write_p = (fat16_write_t)sd_write_block;
*arg_pp = &sd;
return (0);
}
/**
* def start(self)
*/
static mp_obj_t class_sd_start(mp_obj_t self_in)
{
struct class_sd_t *self_p;
self_p = MP_OBJ_TO_PTR(self_in);
if (sd_start(&self_p->drv) != 0) {
nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError,
"sd_start() failed"));
}
return (mp_const_none);
}
void initdtcc()
{
TRACE1(tet_Ttcc, 1, "initdtcc(): start up the servers and log on to each remote system");
tet_ts_startup();
if (
ts_tccinit() < 0 ||
sd_start() < 0 ||
tet_sdlogon() < 0 ||
xd_start(resdirname()) < 0 ||
tet_xdlogon() < 0 ||
tccdlogon() < 0
)
tcc_exit(1);
TRACE1(tet_Ttcc, 1, "initdtcc(): normal RETURN");
}
static int test_init(struct harness_t *harness_p)
{
int res;
#if defined(ARCH_LINUX)
BTASSERT(system("./create_sdcard_linux.sh") == 0);
#endif
BTASSERT(spi_init(&spi,
&spi_device[0],
SPI_DEVICE,
SPI_MODE_MASTER,
SPI_SPEED_250KBPS,
0,
0) == 0);
BTASSERT(sd_init(&sd, &spi) == 0);
BTASSERT((res = sd_start(&sd)) == 0, ", res = %d\r\n", res);
return (0);
}
int main(int argc, char* argv[])
{
if (argc < 3)
{
std::cout << "Usage: " << argv[0] << " [n in MiB]"
#if defined(STXXL_PARALLEL)
<< " [p threads]"
#endif
<< std::endl;
return -1;
}
STXXL_MSG("----------------------------------------");
stxxl::config::get_instance();
std::string Flags = std::string("")
#if STXXL_CHECK_ORDER_IN_SORTS
+ " STXXL_CHECK_ORDER_IN_SORTS"
#endif
#ifdef NDEBUG
+ " NDEBUG"
#endif
#if TINY_PQ
+ " TINY_PQ"
#endif
#if MANUAL_PQ
+ " MANUAL_PQ"
#endif
#if SIDE_PQ
+ " SIDE_PQ"
#endif
#if STXXL_PARALLEL_PQ_MULTIWAY_MERGE_INTERNAL
+ " STXXL_PARALLEL_PQ_MULTIWAY_MERGE_INTERNAL"
#endif
#if STXXL_PARALLEL_PQ_MULTIWAY_MERGE_EXTERNAL
+ " STXXL_PARALLEL_PQ_MULTIWAY_MERGE_EXTERNAL"
#endif
#if STXXL_PARALLEL_PQ_MULTIWAY_MERGE_DELETE_BUFFER
+ " STXXL_PARALLEL_PQ_MULTIWAY_MERGE_DELETE_BUFFER"
#endif
;
STXXL_MSG("Flags:" << Flags);
unsigned long megabytes = atoi(argv[1]);
#if defined(STXXL_PARALLEL_MODE)
int num_threads = atoi(argv[2]);
STXXL_MSG("Threads: " << num_threads);
omp_set_num_threads(num_threads);
__gnu_parallel::_Settings parallel_settings(__gnu_parallel::_Settings::get());
parallel_settings.sort_algorithm = __gnu_parallel::QS_BALANCED;
parallel_settings.sort_splitting = __gnu_parallel::SAMPLING;
parallel_settings.sort_minimal_n = 1000;
parallel_settings.sort_mwms_oversampling = 10;
parallel_settings.merge_splitting = __gnu_parallel::SAMPLING;
parallel_settings.merge_minimal_n = 1000;
parallel_settings.merge_oversampling = 10;
parallel_settings.multiway_merge_algorithm = __gnu_parallel::LOSER_TREE;
parallel_settings.multiway_merge_splitting = __gnu_parallel::EXACT;
parallel_settings.multiway_merge_oversampling = 10;
parallel_settings.multiway_merge_minimal_n = 1000;
parallel_settings.multiway_merge_minimal_k = 2;
__gnu_parallel::_Settings::set(parallel_settings);
#endif
const stxxl::unsigned_type mem_for_queue = 512 * mega;
const stxxl::unsigned_type mem_for_pools = 512 * mega;
#if TINY_PQ
stxxl::STXXL_UNUSED(mem_for_queue);
const unsigned BufferSize1 = 32; // equalize procedure call overheads etc.
const unsigned N = (1 << 9) / sizeof(my_type); // minimal sequence length
const unsigned IntKMAX = 8; // maximal arity for internal mergersq
const unsigned IntLevels = 2; // number of internal levels
const unsigned BlockSize = (4 * mega);
const unsigned ExtKMAX = 8; // maximal arity for external mergers
const unsigned ExtLevels = 2; // number of external levels
typedef stxxl::priority_queue<
stxxl::priority_queue_config<
my_type,
my_cmp,
BufferSize1,
N,
IntKMAX,
IntLevels,
BlockSize,
ExtKMAX,
ExtLevels
>
> pq_type;
#elif MANUAL_PQ
stxxl::STXXL_UNUSED(mem_for_queue);
const unsigned BufferSize1 = 32; // equalize procedure call overheads etc.
const unsigned N = (1 << 20) / sizeof(my_type); // minimal sequence length
const unsigned IntKMAX = 16; // maximal arity for internal mergersq
const unsigned IntLevels = 2; // number of internal levels
const unsigned BlockSize = (4 * mega);
const unsigned ExtKMAX = 32; // maximal arity for external mergers
const unsigned ExtLevels = 2; // number of external levels
typedef stxxl::priority_queue<
stxxl::priority_queue_config<
my_type,
my_cmp,
BufferSize1,
N,
IntKMAX,
IntLevels,
BlockSize,
ExtKMAX,
ExtLevels
>
> pq_type;
#else
const stxxl::uint64 volume = stxxl::uint64(200000) * mega; // in bytes
typedef stxxl::PRIORITY_QUEUE_GENERATOR<my_type, my_cmp, mem_for_queue, volume / sizeof(my_type) / 1024 + 1> gen;
typedef gen::result pq_type;
// BufferSize1 = Config::BufferSize1,
// N = Config::N,
// IntKMAX = Config::IntKMAX,
// IntLevels = Config::IntLevels,
// ExtLevels = Config::ExtLevels,
// Levels = Config::IntLevels + Config::ExtLevels,
// BlockSize = Config::BlockSize,
// ExtKMAX = Config::ExtKMAX
/* STXXL_MSG ( "Blocks fitting into internal memory m: "<<gen::m );
STXXL_MSG ( "X : "<<gen::X ); //maximum number of internal elements //X = B * (settings::k - m) / settings::E,
STXXL_MSG ( "Expected internal memory consumption: "<< (gen::EConsumption / 1048576) << " MiB");*/
#endif
STXXL_MSG("Internal arity: " << pq_type::IntKMAX);
STXXL_MSG("N : " << pq_type::N); //X / (AI * AI)
STXXL_MSG("External arity: " << pq_type::ExtKMAX);
STXXL_MSG("Block size B: " << pq_type::BlockSize);
//EConsumption = X * settings::E + settings::B * AE + ((MaxS_ / X) / AE) * settings::B * 1024
STXXL_MSG("Data type size: " << sizeof(my_type));
STXXL_MSG("");
stxxl::stats_data sd_start(*stxxl::stats::get_instance());
stxxl::timer Timer;
Timer.start();
pq_type p(mem_for_pools / 2, mem_for_pools / 2);
stxxl::int64 nelements = stxxl::int64(megabytes * mega / sizeof(my_type)), i;
STXXL_MSG("Internal memory consumption of the priority queue: " << p.mem_cons() << " B");
STXXL_MSG("Peak number of elements (n): " << nelements);
STXXL_MSG("Max number of elements to contain: " << (stxxl::uint64(pq_type::N) * pq_type::IntKMAX * pq_type::IntKMAX * pq_type::ExtKMAX * pq_type::ExtKMAX));
srand(5);
my_cmp cmp;
my_key_type r, sum_input = 0, sum_output = 0;
my_type least(0), last_least(0);
const my_key_type modulo = 0x10000000;
#if SIDE_PQ
std::priority_queue<my_type, std::vector<my_type>, my_cmp> side_pq;
#endif
my_type side_pq_least;
STXXL_MSG("op-sequence(monotonic pq): ( push, pop, push ) * n");
for (i = 0; i < nelements; ++i)
{
if ((i % mega) == 0)
STXXL_MSG(
std::fixed << std::setprecision(2) << std::setw(5)
<< (100.0 * (double)i / (double)nelements) << "% "
<< "Inserting element " << i << " top() == " << least.key << " @ "
<< std::setprecision(3) << Timer.seconds() << " s"
<< std::setprecision(6) << std::resetiosflags(std::ios_base::floatfield));
//monotone priority queue
r = least.key + rand() % modulo;
sum_input += r;
p.push(my_type(r));
#if SIDE_PQ
side_pq.push(my_type(r));
#endif
least = p.top();
sum_output += least.key;
p.pop();
#if SIDE_PQ
side_pq_least = side_pq.top();
side_pq.pop();
if (!(side_pq_least == least))
STXXL_MSG("Wrong result at " << i << " " << side_pq_least.key << " != " << least.key);
#endif
if (cmp(last_least, least))
{
STXXL_MSG("Wrong order at " << i << " " << last_least.key << " > " << least.key);
}
else
last_least = least;
r = least.key + rand() % modulo;
sum_input += r;
p.push(my_type(r));
#if SIDE_PQ
side_pq.push(my_type(r));
#endif
}
Timer.stop();
STXXL_MSG("Time spent for filling: " << Timer.seconds() << " s");
STXXL_MSG("Internal memory consumption of the priority queue: " << p.mem_cons() << " B");
stxxl::stats_data sd_middle(*stxxl::stats::get_instance());
std::cout << sd_middle - sd_start;
Timer.reset();
Timer.start();
STXXL_MSG("op-sequence(monotonic pq): ( pop, push, pop ) * n");
for (i = 0; i < (nelements); ++i)
{
assert(!p.empty());
least = p.top();
sum_output += least.key;
p.pop();
#if SIDE_PQ
side_pq_least = side_pq.top();
side_pq.pop();
if (!(side_pq_least == least))
{
STXXL_VERBOSE1("" << side_pq_least << " != " << least);
}
#endif
if (cmp(last_least, least))
{
STXXL_MSG("Wrong result at " << i << " " << last_least.key << " > " << least.key);
}
else
last_least = least;
r = least.key + rand() % modulo;
sum_input += r;
p.push(my_type(r));
#if SIDE_PQ
side_pq.push(my_type(r));
#endif
least = p.top();
sum_output += least.key;
p.pop();
#if SIDE_PQ
side_pq_least = side_pq.top();
side_pq.pop();
if (!(side_pq_least == least))
{
STXXL_VERBOSE1("" << side_pq_least << " != " << least);
}
#endif
if (cmp(last_least, least))
{
STXXL_MSG("Wrong result at " << i << " " << last_least.key << " > " << least.key);
}
else
last_least = least;
if ((i % mega) == 0)
STXXL_MSG(
std::fixed << std::setprecision(2) << std::setw(5)
<< (100.0 * (double)i / (double)nelements) << "% "
<< "Popped element " << i << " == " << least.key << " @ "
<< std::setprecision(3) << Timer.seconds() << " s"
<< std::setprecision(6) << std::resetiosflags(std::ios_base::floatfield));
}
STXXL_MSG("Last element " << i << " popped");
Timer.stop();
if (sum_input != sum_output)
STXXL_MSG("WRONG sum! " << sum_input << " - " << sum_output << " = " << (sum_output - sum_input) << " / " << (sum_input - sum_output));
STXXL_MSG("Time spent for removing elements: " << Timer.seconds() << " s");
STXXL_MSG("Internal memory consumption of the priority queue: " << p.mem_cons() << " B");
std::cout << stxxl::stats_data(*stxxl::stats::get_instance()) - sd_middle;
std::cout << *stxxl::stats::get_instance();
assert(sum_input == sum_output);
}
// Big state switch
int uslg_parser_exec(syslog_parser *parser, const syslog_parser_settings *settings, const char *data, size_t length) {
int d_index;
int error = 0;
char next_byte;
for (d_index = 0; d_index < length; d_index++) {
int action = pa_none;
next_byte = data[d_index];
#if DEBUG_OUTPUT
printf("Next byte: %c\n", next_byte);
#endif
// Token state is managed first
if (parser->token_state == ts_before) {
switch (next_byte) {
case ' ':
case '\t':
action = pa_advance;
break;
case '\r':
//case '\n': removed for issues with syslog messages with empty msg field
if (!(parser->flags & F_COUNT_OCTETS)) {
parser->error = SLERR_PREMATURE_MSG_END;
}
break;
default:
set_token_state(parser, ts_read);
action = pa_rehash;
}
} else {
// Parser state
switch (parser->state) {
case s_msg_start:
action = msg_start(parser, settings, next_byte);
break;
case s_octet_count:
action = octet_count(parser, next_byte);
break;
case s_priority_start:
action = priority_start(parser, next_byte);
break;
case s_priority:
action = priority(parser, next_byte);
break;
case s_version:
action = version(parser, next_byte);
break;
case s_timestamp:
action = parse_msg_head_part(parser, s_hostname, next_byte);
break;
case s_hostname:
action = parse_msg_head_part(parser, s_appname, next_byte);
break;
case s_appname:
action = parse_msg_head_part(parser, s_processid, next_byte);
break;
case s_processid:
action = parse_msg_head_part(parser, s_messageid, next_byte);
break;
case s_messageid:
action = parse_msg_head_part(parser, s_sd_start, next_byte);
break;
case s_sd_start:
action = sd_start(parser, settings, next_byte);
break;
case s_sd_element:
action = sd_element(parser, settings, next_byte);
break;
case s_sd_field_start:
action = sd_field_start(parser, next_byte);
break;
case s_sd_field:
action = sd_field(parser, settings, next_byte);
break;
case s_sd_value_start:
action = sd_value_start(parser, next_byte);
break;
case s_sd_value:
action = sd_value(parser, settings, next_byte);
break;
case s_message:
d_index += read_message(parser, settings, data + d_index, length - d_index);
action = pa_rehash;
break;
default:
parser->error = SLERR_BAD_STATE;
}
}
// Upon error, exit the read loop regardless of action
if (parser->error) {
error = parser->error;
uslg_parser_reset(parser);
break;
}
// What action should be taken for this byte
switch (action) {
case pa_advance:
if (parser->flags & F_COUNT_OCTETS) {
parser->octets_remaining--;
} else {
parser->message_length++;
}
break;
case pa_rehash:
d_index--;
break;
}
}
return error;
}
