Example #1
0
 static SockAddr getLocalAddrForBoundSocketFd(int fd) {
     SockAddr result;
     int rc = getsockname(fd, result.raw(), &result.addressSize);
     if (rc != 0) {
         warning() << "Could not resolve local address for socket with fd " << fd << ": " <<
             getAddrInfoStrError(socketGetLastError());
         result = SockAddr();
     }
     return result;
 }
Example #2
0
bool UdpEndPoint::open(const SockAddr& sockaddr, 
                        int backlog,
                        auto_ptr<base::net::HandlerCreatorStrategyBase> handler_stg,
                        auto_ptr<base::net::FilterCreatorStrategyBase> sp_filter_stg,
                        ErrorCode* ec)
{
    int fd = ::socket(sockaddr.getType(), SOCK_DGRAM, 0);
    if (-1 == fd)
        RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);

    ScopedFD spfd(fd);
    if (sockaddr.getType() == AF_UNIX)
    {
        const sockaddr_un& un = sockaddr.as<sockaddr_un>();
        unlink(un.sun_path);   // in case it already exists
    }

    SocketHelper::setNonBlocking(fd, ec);
    SocketHelper::setReusable(fd, ec);

    if (-1 == ::bind(fd, sockaddr.raw(), sockaddr.size()))
        RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);

    boost::shared_ptr<Handler> newhandler;
	if (0 == sockaddr.getPort())
	{
		struct sockaddr_in addr;
		socklen_t len = sizeof(addr);
		if (-1 == getsockname(fd, (struct sockaddr *)&addr, &len))
	        RETURN_NET_ERROR_WITH(ec, errno, "", false);
		SockAddr setaddr(ntohs(addr.sin_port));
	    handler_stg->create(fd, setaddr, sp_reactor_impl_, newhandler);
	}
	else
	    handler_stg->create(fd, sockaddr, sp_reactor_impl_, newhandler);


    if (!newhandler)
        RETURN_NET_ERROR_WITH(ec, errno, "create handler failed" << sockaddr, false);
    newhandler->setConnected();
    
    if (sp_filter_stg.get())
        newhandler->setFilter(auto_ptr<Filter>(sp_filter_stg->create(false, newhandler.get())));

    HandlerScoper<Handler> handler_scoper(newhandler);
    spfd.release();

    if (!sp_reactor_impl_->registerHandler(newhandler, MASK_READ, ec))
        return false;

    handler_scoper.release();
    setUdpHandler(newhandler);
    return true;
}
Example #3
0
bool Acceptor::open(const SockAddr& sockaddr,
        int backlog,
        auto_ptr<HandlerCreatorStrategyBase> handler_creator_stg,
        auto_ptr<FilterCreatorStrategyBase> filter_stg,
        ErrorCode* ec)
{
    if (!handler_creator_stg.get())
        RETURN_ERROR_WITH(ec, E_NET_CREATE_NEW_HANDLER_FAILED, "addr=" << sockaddr, false);

    int fd = ::socket(sockaddr.getType(), SOCK_STREAM, 0);
    if (-1 == fd)
        RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);

    ScopedFD spfd(fd);
    if (sockaddr.getType() == AF_UNIX)
    {
        const sockaddr_un& un = sockaddr.as<sockaddr_un>();
        unlink(un.sun_path);   // in case it already exists
    }

    SocketHelper::setNonBlocking(fd, ec);
    SocketHelper::setReusable(fd, ec);

    if (flag_ & OPT_KEEPALIVE)
        SocketHelper::setKeepAlive(fd, ec);
    if (flag_ & OPT_NODELAY)
        SocketHelper::disableNagle(fd, ec);

    if (-1 == ::bind(fd, sockaddr.raw(), sockaddr.size()))
        RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);

    if (-1 == ::listen(fd, backlog))
        RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);

    boost::shared_ptr<Handler> newhandler;
    newhandler.reset(new AcceptorHandler(fd, sockaddr, sp_reactor_impl_, handler_creator_stg, filter_stg));

    HandlerScoper<Handler> handler_scoper(newhandler);
    spfd.release();

    if (!sp_reactor_impl_->registerHandler(newhandler, MASK_ACCEPT, ec))
        return false;

    LOG(info, "listen on "<< sockaddr);

    handler_scoper.release();
    setAcceptorHandler(newhandler);
    return true;
}
Example #4
0
bool Connector::open(const SockAddr& sockaddr, boost::shared_ptr<Handler>* spout, ErrorCode* ec)
{
    if (!sp_creator_stg_.get())
        RETURN_ERROR_WITH(ec, E_NET_CREATE_NEW_HANDLER_FAILED, "addr=" << sockaddr, false);

    int fd = ::socket(sockaddr.getType(), SOCK_STREAM, 0);
    if (-1 == fd)
        RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);

    ScopedFD spfd(fd);

    SocketHelper::setNonBlocking(fd, ec);
    SocketHelper::setReusable(fd, ec);

    if (flag_ & OPT_KEEPALIVE)
        SocketHelper::setKeepAlive(fd, ec);
    if (flag_ & OPT_NODELAY)
        SocketHelper::disableNagle(fd, ec);

    LOG(info, "connect to " << sockaddr);

    if (-1 == ::connect(fd, sockaddr.raw(), sockaddr.size()))
    {
        if (EINTR != errno && EINPROGRESS != errno)
            RETURN_NET_ERROR_WITH(ec, errno, "addr=" << sockaddr, false);
    }

    boost::shared_ptr<Handler> spnewhandler;
    sp_creator_stg_->create(fd, sockaddr, sp_reactor_impl_, spnewhandler);
    if (!spnewhandler)
        RETURN_ERROR_WITH(ec, E_NET_CREATE_NEW_HANDLER_FAILED, "addr=" << sockaddr, false);

    HandlerScoper<Handler> handler_scoper(spnewhandler);
    spfd.release();

    if (sp_filter_stg_.get())
    	spnewhandler->setFilter(auto_ptr<Filter>(sp_filter_stg_->create(true, spnewhandler.get())));

    if (!sp_reactor_impl_->registerHandler(spnewhandler, MASK_CONNECT|MASK_READ|MASK_WRITE, ec))
        RETURN_IF_ERROR_WITH(ec, false);

    handler_scoper.release();
    if (spout)
        *spout = spnewhandler;
    return true;
}
Example #5
0
    void Listener::initAndListen() {
        vector<SockAddr> mine = ipToAddrs(_ip.c_str(), _port);
        vector<int> socks;
        SOCKET maxfd = 0; // needed for select()

        for (vector<SockAddr>::iterator it=mine.begin(), end=mine.end(); it != end; ++it){
            SockAddr& me = *it;

            SOCKET sock = ::socket(me.getType(), SOCK_STREAM, 0);
            if ( sock == INVALID_SOCKET ) {
                log() << "ERROR: listen(): invalid socket? " << errnoWithDescription() << endl;
            }

            if (me.getType() == AF_UNIX){
#if !defined(_WIN32)
                if (unlink(me.getAddr().c_str()) == -1){
                    int x = errno;
                    if (x != ENOENT){
                        log() << "couldn't unlink socket file " << me << errnoWithDescription(x) << " skipping" << endl;
                        continue;
                    }
                }
#endif
            } else if (me.getType() == AF_INET6) {
                // IPv6 can also accept IPv4 connections as mapped addresses (::ffff:127.0.0.1)
                // That causes a conflict if we don't do set it to IPV6_ONLY
                const int one = 1;
                setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*) &one, sizeof(one));
            }

            prebindOptions( sock );
            
            if ( ::bind(sock, me.raw(), me.addressSize) != 0 ) {
                int x = errno;
                log() << "listen(): bind() failed " << errnoWithDescription(x) << " for socket: " << me.toString() << endl;
                if ( x == EADDRINUSE )
                    log() << "  addr already in use" << endl;
                closesocket(sock);
                return;
            }

            if ( ::listen(sock, 128) != 0 ) {
                log() << "listen(): listen() failed " << errnoWithDescription() << endl;
                closesocket(sock);
                return;
            }

            ListeningSockets::get()->add( sock );

            socks.push_back(sock);
            if (sock > maxfd)
                maxfd = sock;
        }

        static long connNumber = 0;
        struct timeval maxSelectTime;
        while ( ! inShutdown() ) {
            fd_set fds[1];
            FD_ZERO(fds);

            for (vector<int>::iterator it=socks.begin(), end=socks.end(); it != end; ++it){
                FD_SET(*it, fds);
            }

            maxSelectTime.tv_sec = 0;
            maxSelectTime.tv_usec = 10000;
            const int ret = select(maxfd+1, fds, NULL, NULL, &maxSelectTime);
            
            if (ret == 0){
                _elapsedTime += maxSelectTime.tv_usec / 1000;
                continue;
            }
            _elapsedTime += ret; // assume 1ms to grab connection. very rough
            
            if (ret < 0){
                int x = errno;
#ifdef EINTR
                if ( x == EINTR ){
                    log() << "select() signal caught, continuing" << endl;
                    continue;
                }
#endif
                if ( ! inShutdown() )
                    log() << "select() failure: ret=" << ret << " " << errnoWithDescription(x) << endl;
                return;
            }

            for (vector<int>::iterator it=socks.begin(), end=socks.end(); it != end; ++it){
                if (! (FD_ISSET(*it, fds)))
                    continue;

                SockAddr from;
                int s = accept(*it, from.raw(), &from.addressSize);
                if ( s < 0 ) {
                    int x = errno; // so no global issues
                    if ( x == ECONNABORTED || x == EBADF ) {
                        log() << "Listener on port " << _port << " aborted" << endl;
                        return;
                    } if ( x == 0 && inShutdown() ){
                        return;   // socket closed
                    }
                    log() << "Listener: accept() returns " << s << " " << errnoWithDescription(x) << endl;
                    continue;
                } 
                if (from.getType() != AF_UNIX)
                    disableNagle(s);
                if ( _logConnect && ! cmdLine.quiet ) 
                    log() << "connection accepted from " << from.toString() << " #" << ++connNumber << endl;
                accepted(s, from);
            }
        }
    }
Example #6
0
    void Listener::initAndListen() {
        checkTicketNumbers();
        vector<SOCKET> socks;
        set<int> sslSocks;
        
        { // normal sockets
            vector<SockAddr> mine = ipToAddrs(_ip.c_str(), _port, (!cmdLine.noUnixSocket && useUnixSockets()));
            if ( ! _setupSockets( mine , socks ) )
                return;
        }
        
#ifdef MONGO_SSL
        if ( _ssl && _sslPort > 0 ) {
            unsigned prev = socks.size();
            
            vector<SockAddr> mine = ipToAddrs(_ip.c_str(), _sslPort, false );
            if ( ! _setupSockets( mine , socks ) )
                return;
            
            for ( unsigned i=prev; i<socks.size(); i++ ) {
                sslSocks.insert( socks[i] );
            }

        }
#endif

        SOCKET maxfd = 0; // needed for select()
        for ( unsigned i=0; i<socks.size(); i++ ) {
            if ( socks[i] > maxfd )
                maxfd = socks[i];
        }
        
#ifdef MONGO_SSL
        if ( _ssl == 0 ) {
            _logListen( _port , false );
        }
        else if ( _sslPort == 0 ) {
            _logListen( _port , true );
        }
        else {
            // both
            _logListen( _port , false );
            _logListen( _sslPort , true );
        }
#else
        _logListen( _port , false );
#endif

        static long connNumber = 0;
        struct timeval maxSelectTime;
        while ( ! inShutdown() ) {
            fd_set fds[1];
            FD_ZERO(fds);
            
            for (vector<SOCKET>::iterator it=socks.begin(), end=socks.end(); it != end; ++it) {
                FD_SET(*it, fds);
            }

            maxSelectTime.tv_sec = 0;
            maxSelectTime.tv_usec = 10000;
            const int ret = select(maxfd+1, fds, NULL, NULL, &maxSelectTime);

            if (ret == 0) {
#if defined(__linux__)
                _elapsedTime += ( 10000 - maxSelectTime.tv_usec ) / 1000;
#else
                _elapsedTime += 10;
#endif
                continue;
            }

            if (ret < 0) {
                int x = errno;
#ifdef EINTR
                if ( x == EINTR ) {
                    log() << "select() signal caught, continuing" << endl;
                    continue;
                }
#endif
                if ( ! inShutdown() )
                    log() << "select() failure: ret=" << ret << " " << errnoWithDescription(x) << endl;
                return;
            }

#if defined(__linux__)
            _elapsedTime += max(ret, (int)(( 10000 - maxSelectTime.tv_usec ) / 1000));
#else
            _elapsedTime += ret; // assume 1ms to grab connection. very rough
#endif

            for (vector<SOCKET>::iterator it=socks.begin(), end=socks.end(); it != end; ++it) {
                if (! (FD_ISSET(*it, fds)))
                    continue;

                SockAddr from;
                int s = accept(*it, from.raw(), &from.addressSize);
                if ( s < 0 ) {
                    int x = errno; // so no global issues
                    if ( x == ECONNABORTED || x == EBADF ) {
                        log() << "Listener on port " << _port << " aborted" << endl;
                        return;
                    }
                    if ( x == 0 && inShutdown() ) {
                        return;   // socket closed
                    }
                    if( !inShutdown() ) {
                        log() << "Listener: accept() returns " << s << " " << errnoWithDescription(x) << endl;
                        if (x == EMFILE || x == ENFILE) {
                            // Connection still in listen queue but we can't accept it yet
                            error() << "Out of file descriptors. Waiting one second before trying to accept more connections." << warnings;
                            sleepsecs(1);
                        }
                    }
                    continue;
                }
                if (from.getType() != AF_UNIX)
                    disableNagle(s);

#ifdef SO_NOSIGPIPE
                // ignore SIGPIPE signals on osx, to avoid process exit
                const int one = 1;
                setsockopt( s , SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(int));
#endif

                if ( _logConnect && ! cmdLine.quiet ){
                    int conns = connTicketHolder.used()+1;
                    const char* word = (conns == 1 ? " connection" : " connections");
                    log() << "connection accepted from " << from.toString() << " #" << ++connNumber << " (" << conns << word << " now open)" << endl;
                }
                
                boost::shared_ptr<Socket> pnewSock( new Socket(s, from) );
#ifdef MONGO_SSL
                if ( _ssl && ( _sslPort == 0 || sslSocks.count(*it) ) ) {
                    pnewSock->secureAccepted( _ssl );
                }
#endif
                accepted( pnewSock );
            }
        }
    }
Example #7
0
    void Listener::initAndListen() {
        if (!_setupSocketsSuccessful) {
            return;
        }

        for (unsigned i = 0; i < _socks.size(); i++) {
            if (::listen(_socks[i], 128) != 0) {
                error() << "listen(): listen() failed " << errnoWithDescription() << endl;
                return;
            }

            ListeningSockets::get()->add(_socks[i]);
        }

#ifdef MONGO_CONFIG_SSL
        _logListen(_port, _ssl);
#else
        _logListen(_port, false);
#endif

        {
            // Wake up any threads blocked in waitUntilListening()
            stdx::lock_guard<stdx::mutex> lock(_readyMutex);
            _ready = true;
            _readyCondition.notify_all();
        }

        OwnedPointerVector<EventHolder> eventHolders;
        std::unique_ptr<WSAEVENT[]> events(new WSAEVENT[_socks.size()]);
        
        
        // Populate events array with an event for each socket we are watching
        for (size_t count = 0; count < _socks.size(); ++count) {
            EventHolder* ev(new EventHolder);
            eventHolders.mutableVector().push_back(ev);
            events[count] = ev->get();            
        }
            
        while ( ! inShutdown() ) {
            // Turn on listening for accept-ready sockets
            for (size_t count = 0; count < _socks.size(); ++count) {
                int status = WSAEventSelect(_socks[count], events[count], FD_ACCEPT | FD_CLOSE);
                if (status == SOCKET_ERROR) {
                    const int mongo_errno = WSAGetLastError();

                    // During shutdown, we may fail to listen on the socket if it has already
                    // been closed
                    if (inShutdown()) {
                        return;
                    }

                    error() << "Windows WSAEventSelect returned "
                        << errnoWithDescription(mongo_errno) << endl;
                    fassertFailed(16727);
                }
            }
        
            // Wait till one of them goes active, or we time out
            DWORD result = WSAWaitForMultipleEvents(_socks.size(),
                                                    events.get(), 
                                                    FALSE, // don't wait for all the events
                                                    10, // timeout, in ms 
                                                    FALSE); // do not allow I/O interruptions
            if (result == WSA_WAIT_FAILED) {
                const int mongo_errno = WSAGetLastError();
                error() << "Windows WSAWaitForMultipleEvents returned " 
                    << errnoWithDescription(mongo_errno) << endl;
                fassertFailed(16723);
            }
        
            if (result == WSA_WAIT_TIMEOUT) {
                _elapsedTime += 10;
                continue;
            }
            _elapsedTime += 1; // assume 1ms to grab connection. very rough
            
            // Determine which socket is ready
            DWORD eventIndex = result - WSA_WAIT_EVENT_0;
            WSANETWORKEVENTS networkEvents;            
            // Extract event details, and clear event for next pass
            int status = WSAEnumNetworkEvents(_socks[eventIndex],
                                              events[eventIndex], 
                                              &networkEvents);
            if (status == SOCKET_ERROR) {
                const int mongo_errno = WSAGetLastError();
                error() << "Windows WSAEnumNetworkEvents returned " 
                    << errnoWithDescription(mongo_errno) << endl;
                continue;
            }
            
            if (networkEvents.lNetworkEvents & FD_CLOSE) {              
                log() << "listen socket closed" << endl;
                break;
            }
            
            if (!(networkEvents.lNetworkEvents & FD_ACCEPT)) {
                error() << "Unexpected network event: " << networkEvents.lNetworkEvents << endl;
                continue;
            }
            
            int iec = networkEvents.iErrorCode[FD_ACCEPT_BIT];
            if (iec != 0) {                 
                error() << "Windows socket accept did not work:" << errnoWithDescription(iec) 
                        << endl;
                continue;
            }
            
            status = WSAEventSelect(_socks[eventIndex], NULL, 0);
            if (status == SOCKET_ERROR) {
                const int mongo_errno = WSAGetLastError();
                error() << "Windows WSAEventSelect returned " 
                    << errnoWithDescription(mongo_errno) << endl;
                continue;
            }
            
            disableNonblockingMode(_socks[eventIndex]);
            
            SockAddr from;
            int s = accept(_socks[eventIndex], from.raw(), &from.addressSize);
            if ( s < 0 ) {
                int x = errno; // so no global issues
                if (x == EBADF) {
                    log() << "Port " << _port << " is no longer valid" << endl;
                    continue;
                }
                else if (x == ECONNABORTED) {
                    log() << "Listener on port " << _port << " aborted" << endl;
                    continue;
                }
                if ( x == 0 && inShutdown() ) {
                    return;   // socket closed
                }
                if( !inShutdown() ) {
                    log() << "Listener: accept() returns " << s << " " 
                        << errnoWithDescription(x) << endl;
                    if (x == EMFILE || x == ENFILE) {
                        // Connection still in listen queue but we can't accept it yet
                        error() << "Out of file descriptors. Waiting one second before"
                            " trying to accept more connections." << warnings;
                        sleepsecs(1);
                    }
                }
                continue;
            }
            if (from.getType() != AF_UNIX)
                disableNagle(s);

            long long myConnectionNumber = globalConnectionNumber.addAndFetch(1);

            if (_logConnect && !serverGlobalParams.quiet) {
                int conns = globalTicketHolder.used()+1;
                const char* word = (conns == 1 ? " connection" : " connections");
                log() << "connection accepted from " << from.toString() << " #" << myConnectionNumber << " (" << conns << word << " now open)" << endl;
            }
            
            std::shared_ptr<Socket> pnewSock( new Socket(s, from) );
#ifdef MONGO_CONFIG_SSL
            if (_ssl) {
                pnewSock->secureAccepted(_ssl);
            }
#endif
            accepted( pnewSock , myConnectionNumber );
        }
    }
Example #8
0
    void Listener::initAndListen() {
        if (!_setupSocketsSuccessful) {
            return;
        }

        SOCKET maxfd = 0; // needed for select()
        for (unsigned i = 0; i < _socks.size(); i++) {
            if (::listen(_socks[i], 128) != 0) {
                error() << "listen(): listen() failed " << errnoWithDescription() << endl;
                return;
            }

            ListeningSockets::get()->add(_socks[i]);

            if (_socks[i] > maxfd) {
                maxfd = _socks[i];
            }
        }

        if ( maxfd >= FD_SETSIZE ) {
            error() << "socket " << maxfd << " is higher than " << FD_SETSIZE-1 << 
                "; not supported" << warnings;
            return;
        }

#ifdef MONGO_CONFIG_SSL
        _logListen(_port, _ssl);
#else
        _logListen(_port, false);
#endif

        {
            // Wake up any threads blocked in waitUntilListening()
            stdx::lock_guard<stdx::mutex> lock(_readyMutex);
            _ready = true;
            _readyCondition.notify_all();
        }

        struct timeval maxSelectTime;
        while ( ! inShutdown() ) {
            fd_set fds[1];
            FD_ZERO(fds);
            
            for (vector<SOCKET>::iterator it=_socks.begin(), end=_socks.end(); it != end; ++it) {
                FD_SET(*it, fds);
            }

            maxSelectTime.tv_sec = 0;
            maxSelectTime.tv_usec = 10000;
            const int ret = select(maxfd+1, fds, NULL, NULL, &maxSelectTime);

            if (ret == 0) {
#if defined(__linux__)
                _elapsedTime += ( 10000 - maxSelectTime.tv_usec ) / 1000;
#else
                _elapsedTime += 10;
#endif
                continue;
            }

            if (ret < 0) {
                int x = errno;
#ifdef EINTR
                if ( x == EINTR ) {
                    log() << "select() signal caught, continuing" << endl;
                    continue;
                }
#endif
                if ( ! inShutdown() )
                    log() << "select() failure: ret=" << ret << " " << errnoWithDescription(x) << endl;
                return;
            }

#if defined(__linux__)
            _elapsedTime += std::max(ret, (int)(( 10000 - maxSelectTime.tv_usec ) / 1000));
#else
            _elapsedTime += ret; // assume 1ms to grab connection. very rough
#endif

            for (vector<SOCKET>::iterator it=_socks.begin(), end=_socks.end(); it != end; ++it) {
                if (! (FD_ISSET(*it, fds)))
                    continue;
                SockAddr from;
                int s = accept(*it, from.raw(), &from.addressSize);
                if ( s < 0 ) {
                    int x = errno; // so no global issues
                    if (x == EBADF) {
                        log() << "Port " << _port << " is no longer valid" << endl;
                        return;
                    }
                    else if (x == ECONNABORTED) {
                        log() << "Connection on port " << _port << " aborted" << endl;
                        continue;
                    }
                    if ( x == 0 && inShutdown() ) {
                        return;   // socket closed
                    }
                    if( !inShutdown() ) {
                        log() << "Listener: accept() returns " << s << " " << errnoWithDescription(x) << endl;
                        if (x == EMFILE || x == ENFILE) {
                            // Connection still in listen queue but we can't accept it yet
                            error() << "Out of file descriptors. Waiting one second before trying to accept more connections." << warnings;
                            sleepsecs(1);
                        }
                    }
                    continue;
                }
                if (from.getType() != AF_UNIX)
                    disableNagle(s);

#ifdef SO_NOSIGPIPE
                // ignore SIGPIPE signals on osx, to avoid process exit
                const int one = 1;
                setsockopt( s , SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(int));
#endif

                long long myConnectionNumber = globalConnectionNumber.addAndFetch(1);

                if (_logConnect && !serverGlobalParams.quiet) {
                    int conns = globalTicketHolder.used()+1;
                    const char* word = (conns == 1 ? " connection" : " connections");
                    log() << "connection accepted from " << from.toString() << " #" << myConnectionNumber << " (" << conns << word << " now open)" << endl;
                }
                
                std::shared_ptr<Socket> pnewSock( new Socket(s, from) );
#ifdef MONGO_CONFIG_SSL
                if (_ssl) {
                    pnewSock->secureAccepted(_ssl);
                }
#endif
                accepted( pnewSock , myConnectionNumber );
            }
        }
    }