int main(int argc, char *argv[]) {
CURL *curl;
int counter = 0;
if (argc < 2) return 1;
curl = curl_easy_init();
TidyDoc tdoc = tidyCreate();
TidyBuffer output = {0};
tidyOptSetBool(tdoc, TidyXmlOut, yes);
tidyOptSetBool(tdoc, TidyShowWarnings, no);
tidyOptSetInt(tdoc, TidyWrapLen, 0);
for(int i=0; i < 20; i++) {
// tidyBufFree(&output);
tidyBufClear(&output);
// tidyParseFile(tdoc, argv[1]);
tidyParseString(tdoc, getpage(curl,i).c_str());
tidySaveBuffer(tdoc, &output);
// tidySaveFile(tdoc, "tidy_test.xml");
// doc.LoadFile(argv[1]);
// doc.LoadFile("tidy_test.xml");
parseTidyBuf(output, counter);
}
curl_easy_cleanup(curl);
return 0;
}
static int cmd_test(int argc, char **argv)
{
int page;
char fill = 'a';
if (argc < 2) {
printf("usage: %s <page>\n", argv[0]);
return 1;
}
page = getpage(argv[1]);
if (page < 0) {
return 1;
}
for (unsigned i = 0; i < sizeof(page_mem); i++) {
page_mem[i] = (uint8_t)fill++;
if (fill > 'z') {
fill = 'a';
}
}
if (flashpage_write_and_verify(page, page_mem) != FLASHPAGE_OK) {
printf("error verifying the content of page %i\n", page);
return 1;
}
printf("wrote local page buffer to flash page %i at addr %p\n",
page, flashpage_addr(page));
return 0;
}
static DirEntry *
getentry (DirPage0 *page0,
unsigned short num)
{
DirPage1 *page = getpage (page0, num / ENTRIESPERPAGE);
assert (page->header.pg_tag == htons(AFSDIRMAGIC));
return &page->entry[num % ENTRIESPERPAGE];
}
int
sdbm_exists(register DBM *db, datum key)
{
if (db == NULL || bad(key))
return errno = EINVAL, -1;
if (getpage(db, exhash(key)))
return exipair(db->pagbuf, key);
return ioerr(db), -1;
}
datum
sdbm_fetch(register DBM *db, datum key)
{
if (db == NULL || bad(key))
return errno = EINVAL, nullitem;
if (getpage(db, exhash(key)))
return getpair(db->pagbuf, key);
return ioerr(db), nullitem;
}
int area_resize(aspace_t *aspace, int area_id, off_t size)
{
area_t *area;
pagegroup_t *pg;
int ppo;
phys_page_t *pp,*pp0;
uint32 at,p;
if(!(area = rsrc_find_area(area_id))) return ERR_RESOURCE;
pg = area->pgroup;
pp = area->pgroup->pages;
// kprintf("area_resize(%x,%d,%d)",aspace,area_id,size);
if(size <= pg->size*0x1000) return 0;
size = size/0x1000 - pg->size; /* pages to add */
// kprintf("area_resize: %x %x %x + %x",area,pg,pp,size);
at = locate_span(aspace, area->virt_addr + pg->size, size);
if(at != (area->virt_addr + pg->size)) {
kprintf("oops: cannot grow area (%x != %x)",at,area->virt_addr+pg->size);
return ERR_MEMORY;
}
while(pp->next) pp = pp->next;
ppo = pg->size % 6;
pg->size += size;
area->length += size;
area->maxlength += size;
while(size){
if(!ppo){
pp0 = (phys_page_t *) kmalloc(phys_page_t);
pp0->next = NULL;
pp->next = pp0;
pp0->refcount = 0;
pp = pp0;
}
p = getpage();
aspace_map(aspace, p, at, 1, 7);
pp->addr[ppo] = p;
at++;
ppo++;
if(ppo == 6) ppo = 0;
size--;
}
return 0;
}
int
sdbm_store(register DBM *db, datum key, datum val, int flags)
{
int need;
register long hash;
if (db == NULL || bad(key))
return errno = EINVAL, -1;
if (sdbm_rdonly(db))
return errno = EPERM, -1;
need = key.dsize + val.dsize;
/*
* is the pair too big (or too small) for this database ??
*/
if (need < 0 || need > PAIRMAX)
return errno = EINVAL, -1;
if (getpage(db, (hash = exhash(key)))) {
/*
* if we need to replace, delete the key/data pair
* first. If it is not there, ignore.
*/
if (flags == DBM_REPLACE)
(void) delpair(db->pagbuf, key);
#ifdef SEEDUPS
else if (duppair(db->pagbuf, key))
return 1;
#endif
/*
* if we do not have enough room, we have to split.
*/
if (!fitpair(db->pagbuf, need))
if (!makroom(db, hash, need))
return ioerr(db), -1;
/*
* we have enough room or split is successful. insert the key,
* and update the page file.
*/
(void) putpair(db->pagbuf, key, val);
if (lseek(db->pagf, OFF_PAG(db->pagbno), SEEK_SET) < 0
|| write(db->pagf, db->pagbuf, PBLKSIZ) < 0)
return ioerr(db), -1;
/*
* success
*/
return 0;
}
return ioerr(db), -1;
}
int
fdir_creat (fbuf *dir,
const char *name,
AFSFid fid)
{
int ret;
int i;
unsigned npages;
DirPage0 *page0;
DirPage1 *page;
int ind = 0;
unsigned hash_value, next;
assert (dir->len != 0);
page0 = (DirPage0 *)fbuf_buf(dir);
assert (page0);
npages = ntohs(page0->header.pg_pgcount);
if (npages < fbuf_len(dir) / AFSDIR_PAGESIZE)
npages = fbuf_len(dir) / AFSDIR_PAGESIZE;
if (find_entry (page0, name))
return EEXIST;
hash_value = hashentry (name);
next = page0->dheader.hash[hash_value];
for (i = 0; i < npages; ++i) {
page = getpage (page0, i);
ind = add_to_page (page0, page, i, name, fid, next);
if (ind >= 0)
break;
}
if (i == npages) {
ret = create_new_page (&page, dir);
if (ret)
return ret;
page0 = (DirPage0 *)fbuf_buf(dir);
page0->header.pg_pgcount = htons(npages + 1);
if (i < MAXPAGES)
page0->dheader.map[i] = ENTRIESPERPAGE - 1;
ind = add_to_page (page0, page, i, name, fid, next);
assert (ind >= 0);
}
ind += i * ENTRIESPERPAGE;
page0->dheader.hash[hash_value] = htons(ind + 1);
return 0;
}
static int
is_page_empty (DirPage0 *page0, unsigned pageno)
{
DirPage1 *page;
int i;
if (pageno < MAXPAGES)
return page0->dheader.map[pageno] == ENTRIESPERPAGE - 1;
page = getpage (page0, pageno);
if (page->header.pg_bitmap[0] != 1)
return 0;
for (i = 1; i < sizeof(page->header.pg_bitmap); ++i)
if (page->header.pg_bitmap[i] != 0)
return 0;
return 1;
}
static int cmd_erase(int argc, char **argv)
{
int page;
if (argc < 2) {
printf("usage: %s <page>\n", argv[0]);
return 1;
}
page = getpage(argv[1]);
if (page < 0) {
return 1;
}
flashpage_write(page, NULL);
printf("successfully erased page %i (addr %p)\n",
page, flashpage_addr(page));
return 0;
}
static int cmd_read(int argc, char **argv)
{
int page;
if (argc < 2) {
printf("usage: %s <page>\n", argv[0]);
return 1;
}
page = getpage(argv[1]);
if (page < 0) {
return 1;
}
flashpage_read(page, page_mem);
printf("Read flash page %i into local page buffer\n", page);
dump_local();
return 0;
}
/*
* int_stree_new_intleaf
*
* Allocates memory for a new INTLEAF structure.
*
* Parameters: strid - The id of the string containing the new suffix
* to be added to the tree.
* pos - The position of the new suffix in the string.
*
* Returns: The structure or NULL.
*/
STREE_INTLEAF int_stree_new_intleaf(SUFFIX_TREE tree, int strid, int pos)
{
static int pagecount = 0;
static STREE_INTLEAF page = NULL;
STREE_INTLEAF intleaf;
if (page == NULL && (page = (STREE_INTLEAF) getpage(INTLEAF)) == NULL)
return NULL;
intleaf = &page[pagecount];
intleaf->strid = strid;
intleaf->pos = pos;
if (++pagecount == INTLEAFS_PER_PAGE) {
page = NULL;
pagecount = 0;
}
return intleaf;
}
/*
* int_stree_new_node
*
* Allocates memory for a new NODE structure.
*
* Parameters: edgestr - The edge label on the edge to the node.
* edgelen - The edge label's length.
*
* Returns: The structure or NULL.
*/
STREE_NODE int_stree_new_node(SUFFIX_TREE tree, char *edgestr, int edgelen)
{
static int pagecount = 0;
static STREE_NODE page = NULL;
STREE_NODE node;
if (page == NULL && (page = (STREE_NODE) getpage(NODE)) == NULL)
return NULL;
node = &page[pagecount];
node->edgestr = edgestr;
node->edgelen = edgelen;
if (++pagecount == NODES_PER_PAGE) {
page = NULL;
pagecount = 0;
}
return node;
}
static int cmd_dump(int argc, char **argv)
{
int page;
void *addr;
if (argc < 2) {
printf("usage: %s <page>\n", argv[0]);
return 1;
}
page = getpage(argv[1]);
if (page < 0) {
return 1;
}
addr = flashpage_addr(page);
printf("Flash page %i at address %p\n", page, addr);
memdump(addr, FLASHPAGE_SIZE);
return 0;
}
/*
* int_stree_new_leaf
*
* Allocates memory for a new LEAF structure.
*
* Parameters: strid - The id of the string containing the new suffix
* to be added to the tree.
* pos - The position of the new suffix in the string.
* edgestr - The edge label on the edge to the leaf.
* rawedgestr - The raw edge label on the edge to the leaf.
* edgelen - The edge label's length.
*
* Returns: The structure or NULL.
*/
STREE_LEAF int_stree_new_leaf(SUFFIX_TREE tree, int strid, int edgepos)
{
static int pagecount = 0;
static STREE_LEAF page = NULL;
STREE_LEAF leaf;
if (page == NULL && (page = (STREE_LEAF) getpage(LEAF)) == NULL)
return NULL;
leaf = &page[pagecount];
leaf->isaleaf = 1;
leaf->ch = stree_mapch(tree, tree->strings[strid][edgepos]);
leaf->strid = strid;
leaf->pos = edgepos;
if (++pagecount == LEAFS_PER_PAGE) {
page = NULL;
pagecount = 0;
}
return leaf;
}
int
sdbm_delete(register DBM *db, datum key)
{
if (db == NULL || bad(key))
return errno = EINVAL, -1;
if (sdbm_rdonly(db))
return errno = EPERM, -1;
if (getpage(db, exhash(key))) {
if (!delpair(db->pagbuf, key))
return -1;
/*
* update the page file
*/
if (lseek(db->pagf, OFF_PAG(db->pagbno), SEEK_SET) < 0
|| write(db->pagf, db->pagbuf, PBLKSIZ) < 0)
return ioerr(db), -1;
return 0;
}
return ioerr(db), -1;
}
static int cmd_write(int argc, char **argv)
{
int page;
if (argc < 2) {
printf("usage: %s <page>\n", argv[0]);
return 1;
}
page = getpage(argv[1]);
if (page < 0) {
return 1;
}
if (flashpage_write_and_verify(page, page_mem) != FLASHPAGE_OK) {
printf("error: verification for page %i failed\n", page);
return 1;
}
printf("wrote local page buffer to flash page %i at addr %p\n",
page, flashpage_addr(page));
return 0;
}
const uint8_t Burningseries::getlinks(std::vector<Global::episodepair> &episodes)
{
Global::debug("burningseries.cpp");
std::vector<Global::episodepair> pages;
uint8_t ret;
const std::map<const Config::HostingProviders, const Config::providerpair>
&providermap = _cfg.get_providermap();
ret = get_episode_pages(pages);
if (ret != 0)
{
return ret;
}
for (const Global::episodepair &epair : pages)
{
std::map<const Config::HostingProviders, const Config::providerpair>::const_iterator itm;
for (itm = providermap.begin(); itm != providermap.end(); ++itm)
{
if (itm->second.first == epair.second)
{ // Look up provider of found link in providermap
const Config::HostingProviders provider = itm->first;
std::string content = getpage(epair.first);
std::regex reHosterPage(
"(<a href| src)=[\"\'](https?://bs.to/out/.*)[\"\']( target=|></iframe>)");
std::regex reTitle(std::string("<h2 id=\"titleGerman\">(.*)") +
"[[:space:]]+<small id=\"titleEnglish\" lang=\"en\">(.*)</small>");
std::smatch match;
std::string hosterurl;
std::string title;
if (std::regex_search(content, match, reHosterPage))
{
hosterurl = match[2].str();
if (_cfg.get_resolve())
{
resolve_redirect(hosterurl);
}
for (const Config::HostingProviders &provider_ssl : _cfg.get_providers_ssl())
{ // Make sure we use SSL where supported
if (provider_ssl == provider)
{
if (hosterurl.find("https") == std::string::npos)
{ // Replace "http" with "https"
hosterurl = "https" + hosterurl.substr(4, std::string::npos);
}
}
}
}
else
{
std::cerr << "Error extracting stream" << std::endl;
}
if (std::regex_search(content, match, reTitle))
{
if (match[1].str() != "") // German
title = match[1].str();
else if (match[2].str() != "") // English
title = match[2].str();
}
episodes.push_back(Global::episodepair(hosterurl, title));
}
}
}
return 0;
}
/* userland calls */
int area_create(aspace_t *aspace, off_t size, off_t virt, void **addr, uint32 flags)
{
int ppo,i,p=0,at;
area_t *area;
pagegroup_t *pg;
phys_page_t *pp;
size = (size & 0x0FFF) ? (size / 0x1000 + 1) : (size / 0x1000);
if(size < 1) size = 1;
if(flags & AREA_PHYSMAP) {
p = ((uint32) *addr) / 0x1000;
}
/* find a virtaddr */
if(!(at = locate_span(aspace, virt/0x1000, size))){
return ERR_MEMORY;
}
/* create a fresh pagegroup and enough phys_page_t's */
pg = (pagegroup_t *) kmalloc(pagegroup_t);
pg->flags = flags;
pg->refcount = 1;
pg->size = size;
pg->pages = NULL;
list_init(&pg->areas);
for(i=0;i<size;i+=6){
pp = (phys_page_t *) kmalloc(phys_page_t);
pp->refcount = 0;
pp->next = pg->pages;
pg->pages = pp;
};
/* create an area to ref the pagegroup */
area = (area_t *) kmalloc(area_t);
area->pgroup = pg;
area->virt_addr = at;
area->length = size;
area->maxlength = size;
/* link this area into the new pagegroup */
list_add_tail(&pg->areas, area);
/* link this area into the aspace's arealist */
list_add_tail(&aspace->areas, area);
/* check for valid ptr */
*addr = (void *) (at * 0x1000);
/* allocate pages, fill phys_page_t's, and map 'em */
for(ppo=0,pp=pg->pages,i=0;i<size;i++){
if(!(flags & AREA_PHYSMAP)) {
p = getpage();
}
aspace_map(aspace, p, at, 1, 7);
pp->addr[ppo] = p;
ppo++;
if(ppo == 6) {
ppo = 0;
pp = pp->next;
}
if(flags & AREA_PHYSMAP) p++;
at++;
}
/*
* zero extra space in last phys_page_t.
* careful, ppo == 0 and pp == NULL if size == 6.
*/
while((ppo < 6) && (pp != NULL)){
pp->addr[ppo] = 0;
ppo++;
}
/* bind the resource and return the id */
rsrc_bind(&(area->rsrc), RSRC_AREA, current->rsrc.owner);
return area->rsrc.id;
}
int timod_getmsg(unsigned int fd, char *ctl_buf, int ctl_maxlen, s32 *ctl_len,
char *data_buf, int data_maxlen, s32 *data_len, int *flags_p)
{
int error;
int oldflags;
struct file *filp;
struct inode *ino;
struct sol_socket_struct *sock;
struct T_unitdata_ind udi;
mm_segment_t old_fs = get_fs();
long args[6];
char *tmpbuf;
int tmplen;
int (*sys_socketcall)(int, unsigned long *) =
(int (*)(int, unsigned long *))SYS(socketcall);
int (*sys_recvfrom)(int, void *, size_t, unsigned, struct sockaddr *, int *);
SOLD("entry");
SOLDD(("%u %p %d %p %p %d %p %d\n", fd, ctl_buf, ctl_maxlen, ctl_len, data_buf, data_maxlen, data_len, *flags_p));
filp = current->files->fd[fd];
ino = filp->f_dentry->d_inode;
sock = (struct sol_socket_struct *)filp->private_data;
SOLDD(("%p %p\n", sock->pfirst, sock->pfirst ? sock->pfirst->next : NULL));
if ( ctl_maxlen > 0 && !sock->pfirst && ino->u.socket_i.type == SOCK_STREAM
&& sock->state == TS_IDLE) {
SOLD("calling LISTEN");
args[0] = fd;
args[1] = -1;
set_fs(KERNEL_DS);
sys_socketcall(SYS_LISTEN, args);
set_fs(old_fs);
SOLD("LISTEN done");
}
if (!(filp->f_flags & O_NONBLOCK)) {
poll_table wait_table, *wait;
poll_initwait(&wait_table);
wait = &wait_table;
for(;;) {
SOLD("loop");
set_current_state(TASK_INTERRUPTIBLE);
/* ! ( l<0 || ( l>=0 && ( ! pfirst || (flags == HIPRI && pri != HIPRI) ) ) ) */
/* ( ! l<0 && ! ( l>=0 && ( ! pfirst || (flags == HIPRI && pri != HIPRI) ) ) ) */
/* ( l>=0 && ( ! l>=0 || ! ( ! pfirst || (flags == HIPRI && pri != HIPRI) ) ) ) */
/* ( l>=0 && ( l<0 || ( pfirst && ! (flags == HIPRI && pri != HIPRI) ) ) ) */
/* ( l>=0 && ( l<0 || ( pfirst && (flags != HIPRI || pri == HIPRI) ) ) ) */
/* ( l>=0 && ( pfirst && (flags != HIPRI || pri == HIPRI) ) ) */
if (ctl_maxlen >= 0 && sock->pfirst && (*flags_p != MSG_HIPRI || sock->pfirst->pri == MSG_HIPRI))
break;
SOLD("cond 1 passed");
if (
#if 1
*flags_p != MSG_HIPRI &&
#endif
((filp->f_op->poll(filp, wait) & POLLIN) ||
(filp->f_op->poll(filp, NULL) & POLLIN) ||
signal_pending(current))
) {
break;
}
if( *flags_p == MSG_HIPRI ) {
SOLD("avoiding lockup");
break ;
}
if(wait_table.error) {
SOLD("wait-table error");
poll_freewait(&wait_table);
return wait_table.error;
}
SOLD("scheduling");
schedule();
}
SOLD("loop done");
current->state = TASK_RUNNING;
poll_freewait(&wait_table);
if (signal_pending(current)) {
SOLD("signal pending");
return -EINTR;
}
}
if (ctl_maxlen >= 0 && sock->pfirst) {
struct T_primsg *it = sock->pfirst;
int l = min_t(int, ctl_maxlen, it->length);
SCHECK_MAGIC((char*)((u64)(((char *)&it->type)+sock->offset+it->length+7)&~7),MKCTL_MAGIC);
SOLD("purting ctl data");
if(copy_to_user(ctl_buf,
(char*)&it->type + sock->offset, l))
return -EFAULT;
SOLD("pur it");
if(put_user(l, ctl_len))
return -EFAULT;
SOLD("set ctl_len");
*flags_p = it->pri;
it->length -= l;
if (it->length) {
SOLD("more ctl");
sock->offset += l;
return MORECTL;
} else {
SOLD("removing message");
sock->pfirst = it->next;
if (!sock->pfirst)
sock->plast = NULL;
SOLDD(("getmsg kfree %016lx->%016lx\n", it, sock->pfirst));
mykfree(it);
sock->offset = 0;
SOLD("ctl done");
return 0;
}
}
*flags_p = 0;
if (ctl_maxlen >= 0) {
SOLD("ACCEPT perhaps?");
if (ino->u.socket_i.type == SOCK_STREAM && sock->state == TS_IDLE) {
struct T_conn_ind ind;
char *buf = getpage();
int len = BUF_SIZE;
SOLD("trying ACCEPT");
if (put_user(ctl_maxlen - sizeof(ind), ctl_len))
return -EFAULT;
args[0] = fd;
args[1] = (long)buf;
args[2] = (long)&len;
oldflags = filp->f_flags;
filp->f_flags |= O_NONBLOCK;
SOLD("calling ACCEPT");
set_fs(KERNEL_DS);
error = sys_socketcall(SYS_ACCEPT, args);
set_fs(old_fs);
filp->f_flags = oldflags;
if (error < 0) {
SOLD("some error");
putpage(buf);
return error;
}
if (error) {
SOLD("connect");
putpage(buf);
if (sizeof(ind) > ctl_maxlen) {
SOLD("generating CONN_IND");
ind.PRIM_type = T_CONN_IND;
ind.SRC_length = len;
ind.SRC_offset = sizeof(ind);
ind.OPT_length = ind.OPT_offset = 0;
ind.SEQ_number = error;
if(copy_to_user(ctl_buf, &ind, sizeof(ind))||
put_user(sizeof(ind)+ind.SRC_length,ctl_len))
return -EFAULT;
SOLD("CONN_IND created");
}
if (data_maxlen >= 0)
put_user(0, data_len);
SOLD("CONN_IND done");
return 0;
}
if (len>ctl_maxlen) {
SOLD("data don't fit");
putpage(buf);
return -EFAULT; /* XXX - is this ok ? */
}
if(copy_to_user(ctl_buf,buf,len) || put_user(len,ctl_len)){
SOLD("can't copy data");
putpage(buf);
return -EFAULT;
}
SOLD("ACCEPT done");
putpage(buf);
}
}
SOLD("checking data req");
if (data_maxlen <= 0) {
if (data_maxlen == 0)
put_user(0, data_len);
if (ctl_maxlen >= 0)
put_user(0, ctl_len);
return -EAGAIN;
}
SOLD("wants data");
if (ctl_maxlen > sizeof(udi) && sock->state == TS_IDLE) {
SOLD("udi fits");
tmpbuf = ctl_buf + sizeof(udi);
tmplen = ctl_maxlen - sizeof(udi);
} else {
SOLD("udi does not fit");
tmpbuf = NULL;
tmplen = 0;
}
if (put_user(tmplen, ctl_len))
return -EFAULT;
SOLD("set ctl_len");
oldflags = filp->f_flags;
filp->f_flags |= O_NONBLOCK;
SOLD("calling recvfrom");
sys_recvfrom = (int (*)(int, void *, size_t, unsigned, struct sockaddr *, int *))SYS(recvfrom);
error = sys_recvfrom(fd, data_buf, data_maxlen, 0, (struct sockaddr*)tmpbuf, ctl_len);
filp->f_flags = oldflags;
if (error < 0)
return error;
SOLD("error >= 0" ) ;
if (error && ctl_maxlen > sizeof(udi) && sock->state == TS_IDLE) {
SOLD("generating udi");
udi.PRIM_type = T_UNITDATA_IND;
get_user(udi.SRC_length, ctl_len);
udi.SRC_offset = sizeof(udi);
udi.OPT_length = udi.OPT_offset = 0;
copy_to_user(ctl_buf, &udi, sizeof(udi));
put_user(sizeof(udi)+udi.SRC_length, ctl_len);
SOLD("udi done");
} else
put_user(0, ctl_len);
put_user(error, data_len);
SOLD("done");
return 0;
}
int timod_putmsg(unsigned int fd, char *ctl_buf, int ctl_len,
char *data_buf, int data_len, int flags)
{
int ret, error, terror;
char *buf;
struct file *filp;
struct inode *ino;
struct sol_socket_struct *sock;
mm_segment_t old_fs = get_fs();
long args[6];
int (*sys_socketcall)(int, unsigned long *) =
(int (*)(int, unsigned long *))SYS(socketcall);
int (*sys_sendto)(int, void *, size_t, unsigned, struct sockaddr *, int) =
(int (*)(int, void *, size_t, unsigned, struct sockaddr *, int))SYS(sendto);
filp = current->files->fd[fd];
ino = filp->f_dentry->d_inode;
sock = (struct sol_socket_struct *)filp->private_data;
SOLD("entry");
if (get_user(ret, (int *)A(ctl_buf)))
return -EFAULT;
switch (ret) {
case T_BIND_REQ:
{
struct T_bind_req req;
SOLDD(("bind %016lx(%016lx)\n", sock, filp));
SOLD("T_BIND_REQ");
if (sock->state != TS_UNBND) {
timod_error(fd, T_BIND_REQ, TOUTSTATE, 0);
return 0;
}
SOLD("state ok");
if (copy_from_user(&req, ctl_buf, sizeof(req))) {
timod_error(fd, T_BIND_REQ, TSYSERR, EFAULT);
return 0;
}
SOLD("got ctl req");
if (req.ADDR_offset && req.ADDR_length) {
if (req.ADDR_length > BUF_SIZE) {
timod_error(fd, T_BIND_REQ, TSYSERR, EFAULT);
return 0;
}
SOLD("req size ok");
buf = getpage();
if (copy_from_user(buf, ctl_buf + req.ADDR_offset, req.ADDR_length)) {
timod_error(fd, T_BIND_REQ, TSYSERR, EFAULT);
putpage(buf);
return 0;
}
SOLD("got ctl data");
args[0] = fd;
args[1] = (long)buf;
args[2] = req.ADDR_length;
SOLD("calling BIND");
set_fs(KERNEL_DS);
error = sys_socketcall(SYS_BIND, args);
set_fs(old_fs);
putpage(buf);
SOLD("BIND returned");
} else
error = 0;
if (!error) {
struct T_primsg *it;
if (req.CONIND_number) {
args[0] = fd;
args[1] = req.CONIND_number;
SOLD("calling LISTEN");
set_fs(KERNEL_DS);
error = sys_socketcall(SYS_LISTEN, args);
set_fs(old_fs);
SOLD("LISTEN done");
}
it = timod_mkctl(sizeof(struct T_bind_ack)+sizeof(struct sockaddr));
if (it) {
struct T_bind_ack *ack;
ack = (struct T_bind_ack *)&it->type;
ack->PRIM_type = T_BIND_ACK;
ack->ADDR_offset = sizeof(*ack);
ack->ADDR_length = sizeof(struct sockaddr);
ack->CONIND_number = req.CONIND_number;
args[0] = fd;
args[1] = (long)(ack+sizeof(*ack));
args[2] = (long)&ack->ADDR_length;
set_fs(KERNEL_DS);
sys_socketcall(SYS_GETSOCKNAME,args);
set_fs(old_fs);
sock->state = TS_IDLE;
timod_ok(fd, T_BIND_REQ);
timod_queue_end(fd, it);
SOLD("BIND done");
return 0;
}
}
SOLD("some error");
switch (error) {
case -EINVAL:
terror = TOUTSTATE;
error = 0;
break;
case -EACCES:
terror = TACCES;
error = 0;
break;
case -EADDRNOTAVAIL:
case -EADDRINUSE:
terror = TNOADDR;
error = 0;
break;
default:
terror = TSYSERR;
break;
}
timod_error(fd, T_BIND_REQ, terror, -error);
SOLD("BIND done");
return 0;
}
case T_CONN_REQ:
{
struct T_conn_req req;
unsigned short oldflags;
struct T_primsg *it;
SOLD("T_CONN_REQ");
if (sock->state != TS_UNBND && sock->state != TS_IDLE) {
timod_error(fd, T_CONN_REQ, TOUTSTATE, 0);
return 0;
}
SOLD("state ok");
if (copy_from_user(&req, ctl_buf, sizeof(req))) {
timod_error(fd, T_CONN_REQ, TSYSERR, EFAULT);
return 0;
}
SOLD("got ctl req");
if (ctl_len > BUF_SIZE) {
timod_error(fd, T_CONN_REQ, TSYSERR, EFAULT);
return 0;
}
SOLD("req size ok");
buf = getpage();
if (copy_from_user(buf, ctl_buf, ctl_len)) {
timod_error(fd, T_CONN_REQ, TSYSERR, EFAULT);
putpage(buf);
return 0;
}
#ifdef DEBUG_SOLARIS
{
char * ptr = buf;
int len = ctl_len;
printk("returned data (%d bytes): ",len);
while( len-- ) {
if (!(len & 7))
printk(" ");
printk("%02x",(unsigned char)*ptr++);
}
printk("\n");
}
#endif
SOLD("got ctl data");
args[0] = fd;
args[1] = (long)buf+req.DEST_offset;
args[2] = req.DEST_length;
oldflags = filp->f_flags;
filp->f_flags &= ~O_NONBLOCK;
SOLD("calling CONNECT");
set_fs(KERNEL_DS);
error = sys_socketcall(SYS_CONNECT, args);
set_fs(old_fs);
filp->f_flags = oldflags;
SOLD("CONNECT done");
if (!error) {
struct T_conn_con *con;
SOLD("no error");
it = timod_mkctl(ctl_len);
if (!it) {
putpage(buf);
return -ENOMEM;
}
con = (struct T_conn_con *)&it->type;
#ifdef DEBUG_SOLARIS
{
char * ptr = buf;
int len = ctl_len;
printk("returned data (%d bytes): ",len);
while( len-- ) {
if (!(len & 7))
printk(" ");
printk("%02x",(unsigned char)*ptr++);
}
printk("\n");
}
#endif
memcpy(con, buf, ctl_len);
SOLD("copied ctl_buf");
con->PRIM_type = T_CONN_CON;
sock->state = TS_DATA_XFER;
} else {
struct T_discon_ind *dis;
SOLD("some error");
it = timod_mkctl(sizeof(*dis));
if (!it) {
putpage(buf);
return -ENOMEM;
}
SOLD("got primsg");
dis = (struct T_discon_ind *)&it->type;
dis->PRIM_type = T_DISCON_IND;
dis->DISCON_reason = -error; /* FIXME: convert this as in iABI_errors() */
dis->SEQ_number = 0;
}
putpage(buf);
timod_ok(fd, T_CONN_REQ);
it->pri = 0;
timod_queue_end(fd, it);
SOLD("CONNECT done");
return 0;
}
case T_OPTMGMT_REQ:
{
struct T_optmgmt_req req;
SOLD("OPTMGMT_REQ");
if (copy_from_user(&req, ctl_buf, sizeof(req)))
return -EFAULT;
SOLD("got req");
return timod_optmgmt(fd, req.MGMT_flags,
req.OPT_offset > 0 ? ctl_buf + req.OPT_offset : NULL,
req.OPT_length, 1);
}
case T_UNITDATA_REQ:
{
struct T_unitdata_req req;
int err;
SOLD("T_UNITDATA_REQ");
if (sock->state != TS_IDLE && sock->state != TS_DATA_XFER) {
timod_error(fd, T_CONN_REQ, TOUTSTATE, 0);
return 0;
}
SOLD("state ok");
if (copy_from_user(&req, ctl_buf, sizeof(req))) {
timod_error(fd, T_CONN_REQ, TSYSERR, EFAULT);
return 0;
}
SOLD("got ctl req");
#ifdef DEBUG_SOLARIS
{
char * ptr = ctl_buf+req.DEST_offset;
int len = req.DEST_length;
printk("socket address (%d bytes): ",len);
while( len-- ) {
char c;
if (get_user(c,ptr))
printk("??");
else
printk("%02x",(unsigned char)c);
ptr++;
}
printk("\n");
}
#endif
err = sys_sendto(fd, data_buf, data_len, 0, req.DEST_length > 0 ? (struct sockaddr*)(ctl_buf+req.DEST_offset) : NULL, req.DEST_length);
if (err == data_len)
return 0;
if(err >= 0) {
printk("timod: sendto failed to send all the data\n");
return 0;
}
timod_error(fd, T_CONN_REQ, TSYSERR, -err);
return 0;
}
default:
printk(KERN_INFO "timod_putmsg: unsupported command %u.\n", ret);
break;
}
return -EINVAL;
}
static int timod_optmgmt(unsigned int fd, int flag, char *opt_buf, int opt_len, int do_ret)
{
int error, failed;
int ret_space, ret_len;
long args[5];
char *ret_pos,*ret_buf;
int (*sys_socketcall)(int, unsigned long *) =
(int (*)(int, unsigned long *))SYS(socketcall);
mm_segment_t old_fs = get_fs();
SOLD("entry");
SOLDD(("fd %u flg %u buf %p len %u doret %u",fd,flag,opt_buf,opt_len,do_ret));
if (!do_ret && (!opt_buf || opt_len <= 0))
return 0;
SOLD("getting page");
ret_pos = ret_buf = getpage();
ret_space = BUF_SIZE;
ret_len = 0;
error = failed = 0;
SOLD("looping");
while(opt_len >= sizeof(struct opthdr)) {
struct opthdr *opt;
int orig_opt_len;
SOLD("loop start");
opt = (struct opthdr *)ret_pos;
if (ret_space < sizeof(struct opthdr)) {
failed = TSYSERR;
break;
}
SOLD("getting opthdr");
if (copy_from_user(opt, opt_buf, sizeof(struct opthdr)) ||
opt->len > opt_len) {
failed = TBADOPT;
break;
}
SOLD("got opthdr");
if (flag == T_NEGOTIATE) {
char *buf;
SOLD("handling T_NEGOTIATE");
buf = ret_pos + sizeof(struct opthdr);
if (ret_space < opt->len + sizeof(struct opthdr) ||
copy_from_user(buf, opt_buf+sizeof(struct opthdr), opt->len)) {
failed = TSYSERR;
break;
}
SOLD("got optdata");
args[0] = fd;
args[1] = opt->level;
args[2] = opt->name;
args[3] = (long)buf;
args[4] = opt->len;
SOLD("calling SETSOCKOPT");
set_fs(KERNEL_DS);
error = sys_socketcall(SYS_SETSOCKOPT, args);
set_fs(old_fs);
if (error) {
failed = TBADOPT;
break;
}
SOLD("SETSOCKOPT ok");
}
orig_opt_len = opt->len;
opt->len = ret_space - sizeof(struct opthdr);
if (opt->len < 0) {
failed = TSYSERR;
break;
}
args[0] = fd;
args[1] = opt->level;
args[2] = opt->name;
args[3] = (long)(ret_pos+sizeof(struct opthdr));
args[4] = (long)&opt->len;
SOLD("calling GETSOCKOPT");
set_fs(KERNEL_DS);
error = sys_socketcall(SYS_GETSOCKOPT, args);
set_fs(old_fs);;
if (error) {
failed = TBADOPT;
break;
}
SOLD("GETSOCKOPT ok");
ret_space -= sizeof(struct opthdr) + opt->len;
ret_len += sizeof(struct opthdr) + opt->len;
ret_pos += sizeof(struct opthdr) + opt->len;
opt_len -= sizeof(struct opthdr) + orig_opt_len;
opt_buf += sizeof(struct opthdr) + orig_opt_len;
SOLD("loop end");
}
SOLD("loop done");
if (do_ret) {
SOLD("generating ret msg");
if (failed)
timod_error(fd, T_OPTMGMT_REQ, failed, -error);
else {
struct T_primsg *it;
it = timod_mkctl(sizeof(struct T_optmgmt_ack) + ret_len);
if (it) {
struct T_optmgmt_ack *ack =
(struct T_optmgmt_ack *)&it->type;
SOLD("got primsg");
ack->PRIM_type = T_OPTMGMT_ACK;
ack->OPT_length = ret_len;
ack->OPT_offset = sizeof(struct T_optmgmt_ack);
ack->MGMT_flags = (failed ? T_FAILURE : flag);
memcpy(((char*)ack)+sizeof(struct T_optmgmt_ack),
ret_buf, ret_len);
timod_queue(fd, it);
}
}
}
SOLDD(("put_page %p\n", ret_buf));
putpage(ret_buf);
SOLD("done");
return 0;
}
int
fdir_remove (fbuf *dir,
const char *name,
AFSFid *fid)
{
int i;
unsigned len = dir->len;
DirPage0 *page0;
DirPage1 *page;
unsigned hash_value;
DirEntry *entry = NULL;
DirEntry *prev_entry;
unsigned pageno;
int found;
unsigned npages;
page0 = (DirPage0 *)fbuf_buf(dir);
npages = ntohs(page0->header.pg_pgcount);
if (npages < len / AFSDIR_PAGESIZE)
npages = len / AFSDIR_PAGESIZE;
hash_value = hashentry (name);
i = ntohs(page0->dheader.hash[hash_value]);
found = i == 0;
prev_entry = NULL;
while (!found) {
entry = getentry (page0, i - 1);
if (strcmp (entry->name, name) == 0) {
found = TRUE;
} else {
i = ntohs(entry->next);
if (i == 0)
found = TRUE;
prev_entry = entry;
}
}
if (i == 0)
return ENOENT;
else {
if (fid) {
fid->Vnode = ntohl(entry->fid.Vnode);
fid->Unique = ntohl(entry->fid.Unique);
}
if (prev_entry == NULL)
page0->dheader.hash[hash_value] = entry->next;
else
prev_entry->next = entry->next;
pageno = (i - 1) / ENTRIESPERPAGE;
page = getpage (page0, pageno);
remove_from_page (page0, page, pageno, (i - 1) % ENTRIESPERPAGE);
if (pageno == npages - 1
&& is_page_empty (page0, pageno)) {
do {
len -= AFSDIR_PAGESIZE;
--pageno;
--npages;
} while(is_page_empty(page0, pageno));
page0->header.pg_pgcount = htons(npages);
fbuf_truncate (dir, len);
}
return 0;
}
}
int
fdir_readdir (fbuf *the_fbuf,
fdir_readdir_func func,
void *arg,
VenusFid dir,
uint32_t *offset)
{
DirPage0 *page0;
unsigned i, j;
VenusFid fid;
unsigned len = fbuf_len(the_fbuf);
unsigned npages;
unsigned first_slot;
int ret;
page0 = (DirPage0 *)fbuf_buf(the_fbuf);
assert (page0);
npages = ntohs(page0->header.pg_pgcount);
if (npages < len / AFSDIR_PAGESIZE)
npages = len / AFSDIR_PAGESIZE;
if (offset && *offset) {
i = *offset / ENTRIESPERPAGE;
first_slot = *offset % ENTRIESPERPAGE;
assert(i > 0 || first_slot >= 12);
} else {
i = 0;
first_slot = 12;
}
for (; i < npages; ++i) {
DirPage1 *page = getpage (page0, i);
for (j = first_slot; j < ENTRIESPERPAGE - 1; ++j) {
if (first_slotp (page, j)) {
DirEntry *entry = &page->entry[j];
if (entry->flag != AFSDIR_FIRST)
continue;
fid.Cell = dir.Cell;
fid.fid.Volume = dir.fid.Volume;
fid.fid.Vnode = ntohl (entry->fid.Vnode);
fid.fid.Unique = ntohl (entry->fid.Unique);
ret = (*func)(&fid, entry->name, arg);
if (ret) {
if (ret > 0)
j++; /* look at next entry next time */
goto done;
}
j += additional_entries (entry->name);
}
}
first_slot = 0;
}
done:
if (offset)
*offset = i * ENTRIESPERPAGE + j;
return 0;
}
const uint8_t Burningseries::get_episode_pages(std::vector<Global::episodepair> &pages)
{
std::string provider_re = "(";
std::vector<Config::HostingProviders>::const_iterator it;
const std::vector<Config::HostingProviders> &providers = _cfg.get_providers();
const std::map<const Config::HostingProviders, const Config::providerpair>
&providermap = _cfg.get_providermap();
const std::array<uint16_t, 2> &seasonrange = _cfg.get_season_range();
std::array<uint16_t, 2> episoderange = _cfg.get_episode_range();
std::string url = _cfg.get_url();
std::string content;
const uint8_t &use_current_episode = _cfg.get_use_current_episode();
for (it = providers.begin(); it != providers.end(); ++it)
{ // Build regular expression for all supported streaming providers
if (it != providers.begin())
{ // Add | unless it is the first match
provider_re += "|";
}
provider_re += providermap.at(*it).first;
}
provider_re += ")";
for (uint16_t season = seasonrange[0]; season <= seasonrange[1]; ++season)
{
if (season != 0)
{ // A season range was selected
// NOTE: If season is higher than available seasons, season 1 is returned
std::regex reSeries("(https://bs.to/serie/[^/]*/).*");
std::smatch match;
// Generate URL for each season
if (std::regex_search(url, match, reSeries))
{
url = match[1].str() + std::to_string(season);
}
else
{
url = url + "/" + std::to_string(season);
}
content = getpage(url);
}
else
{ // If no season range was selected, use supplied URL
content = getpage(url);
}
if (use_current_episode != 0b00)
{ // replace 'c's with episode numbers
std::regex reEpisode("https://bs.to/serie/[^/]*/[[:digit:]]+/([[:digit:]]+)-.*");
std::smatch match;
if (std::regex_search(url, match, reEpisode))
{
if ((use_current_episode & 0b01) != 0)
{ // Start from current episode
episoderange[0] = std::stoi(match[1].str());
}
if ((use_current_episode & 0b10) != 0)
{ // End with current episode
episoderange[1] = std::stoi(match[1].str());
}
}
else
{
std::cerr << "Error: Could not extract current episode." << std::endl;
return 3;
}
}
std::regex reEpisodePage("href=\"(serie/.*/[[:digit:]]+/([[:digit:]]+)-.*/(" +
provider_re + "))\">(" + provider_re + ")?</a>");
std::sregex_iterator it_re(content.begin(), content.end(), reEpisodePage);
std::sregex_iterator it_re_end;
if (it_re == it_re_end)
{ // No matches
std::cerr << "Error: No episodes found" << std::endl;
return 3;
}
uint16_t episode = 1;
while (it_re != it_re_end)
{ // 1 == link, 2 == episode, 3 == provider
uint16_t found_episode = std::stoi((*it_re)[2]);
if (episode == 1 && episoderange[0] > 1)
{ // Set episode to the right starting value if episode range is specified
episode = episoderange[0];
}
if (found_episode >= episoderange[0] &&
found_episode <= episoderange[1] &&
found_episode >= episode)
{
if (found_episode > episode)
{ // If found episode is > expected episode, we are missing 1 ore more episode(s)
std::cerr << "Error: Could not get URL for episode " << episode;
if ((episode + 1) != found_episode)
{ // If more than 1 episode is missing
std::cerr << "-" << (found_episode - 1);
}
std::cerr << "." << std::endl;
}
// Put URL and provider to episode pages into &pages
pages.push_back(Global::episodepair("https://bs.to/" + (*it_re)[1].str(),
(*it_re)[3].str()));
episode = found_episode + 1;
}
else if (found_episode > episoderange[1])
{
break;
}
++it_re;
} // Iterating through matches
}
return 0;
}
int main(int argc,char* argv[])
{
//wget() for validating URL --without proxy--
if(!system("wget --spider argv[1]"))
printf("Valid URL\n");
else
printf("Invalid URL\n");
struct hash h[HASH_size];
for(int i=0;i<HASH_size;i++)
{
h[i].head1=NULL;
h[i].tail1=NULL;
h[i].countlinks=0;
}
int links=0;
int *totallinks=&links;
for(int k=0;k<174;k++)
{
int counter=k;
if(counter==0)
getpage(argv[1]);
else
{
struct node *t=head;
while(counter!=1)
{
t=t->next;
counter--;
}
getpage(t->u.str);
}
//to get the size of file in which wget gets the html code
struct stat st; //variable which will count length of file.
printf(" file exits if 0 and do not exists if not equal to 0::%d\n", stat("/home/prince/study/crawler/files/temp.txt",&st));
long int file_size=st.st_size;
printf(" size of file is %ld\n",file_size);
//allocating size to filevalistr
char *filevalistr=(char *)malloc(sizeof(char)*(file_size+1));
//copying html file data into filevalistr
int i=0;
FILE *fptr=fopen("/home/prince/study/crawler/files/temp.txt","r");
if(fptr == NULL)
{
printf("file not found\n");
exit(1);
}
char ch;
ch=getc(fptr);
while(ch != EOF)
{
filevalistr[i++]=ch;
ch=getc(fptr);
}
//filevalistr[i++]='\0';
fclose(fptr);
char p[50]={0};
// printf("hey p is%s\n",p);
int fileno=k;
createpathforfile(p,fileno);
fptr=fopen(p,"w");
fprintf(fptr,"%s",filevalistr);
//finalsize mein finalstring k andar jitni rows store hai unka size hai
int finalsize;
int *ptr=&finalsize;
//printf("filevali str is \n %s\n",filevalistr);
char **finalstorestr=extracturls(filevalistr,ptr);
//printfinalstore(finalsize,finalstorestr);
int key;
for(int i=0;i<finalsize;i++)
{
key=hash(finalstorestr[i]);
inserthash(h,key,finalstorestr[i],3,totallinks);
}
}
printlist();
printf("total no of links are %d\n",*totallinks);
}
