void KuhnPoker::performAction(action_t action) {
assert(isValidAction(action));
m_action = action;
// If the agent did not call the environments bet then the agent loses
if(m_action == aPass && m_env_action == aBet) {
m_reward = rPassLoss;
reset();
return;
}
// If the environment passed and the agent bet, then the environment has
// a chance to change its mind.
if(m_action == aBet && m_env_action == aPass) {
if(m_env_card == oQueen && rand01() < cBetProbQueen) {
m_env_action = aBet; // Bet with cBetProbQueen probability on queen
} else if(m_env_card == oKing) {
m_env_action = aBet; // Always bet on king
} else {
// Environment continues to pass, so agent wins
m_reward = rPassWin;
reset();
return;
}
}
// Players have bet the same amount, winner has highest card
bool agent_wins = m_env_card == oJack
|| (m_env_card == oQueen && m_agent_card == oKing);
if(agent_wins)
m_reward = (m_env_action == aBet ? rBetWin : rPassWin);
else
m_reward = (m_action == aBet ? rBetLoss : rPassLoss);
reset();
}
// Execute agent's action.
void ExtendedTiger::performAction(const action_t action) {
assert(isValidAction(action));
m_action = action;
// Unless explicitly accounted for, action is invalid.
m_observation = oNull; // Some valid actions also have null observation.
m_reward = rInvalid;
if (action == aListen && m_sitting) {
// Listen while sitting down. Observe door hiding tiger with probability
// m_listen_accuracy otherwise observe other door.
m_observation = rand01() < m_listen_accuracy ? m_tiger : m_gold;
m_reward = rListen;
} else if (action == aLeft && !m_sitting) {
// Open left door while standing. Get reward based on what was behind
// the door. Reseat agent and reallocate tiger and gold.
m_reward = (m_tiger == oLeft ? rTiger : rGold);
reset();
} else if (action == aRight && !m_sitting) {
// Open right door while standing. Get reward based on what was behind
// the door. Reseat agent and reallocate tiger and gold.
m_reward = (m_tiger == oRight ? rTiger : rGold);
reset();
} else if (action == aStand && m_sitting) {
// Stand from a sitting position. Get reward for standing.
m_reward = rStand;
m_sitting = false;
}
}
void ExternalEnvironment::performAction(const action_t action) {
assert(isValidAction(action));
protocol_send_action(m_sockfd, action);
protocol_recv_percept(m_sockfd, &m_observation, &m_reward);
assert(isValidObservation(m_observation));
assert(isValidReward(m_reward));
}
action_c NullActionAbstraction::get_actions(const Game *game,
const State &state) {
action_c actions(MAX_ABSTRACT_ACTIONS);
int num_actions = 0;
bool error = false;
for (int a = 0; a < NUM_ACTION_TYPES; ++a) {
Action action;
action.type = (ActionType)a;
action.size = 0;
if (action.type == a_raise) {
int32_t min_raise_size;
int32_t max_raise_size;
if (raiseIsValid(game, &state, &min_raise_size, &max_raise_size)) {
if (num_actions + (max_raise_size - min_raise_size + 1) >
MAX_ABSTRACT_ACTIONS) {
error = true;
break;
}
for (int s = min_raise_size; s <= max_raise_size; ++s) {
actions[num_actions] = action;
actions[num_actions].size = s;
++num_actions;
}
}
} else if (isValidAction(game, &state, 0, &action)) {
/* If you hit this assert, there are too many abstract actions allowed.
* Either coarsen the betting abstraction or increase
* MAX_ABSTRACT_ACTIONS
* in constants.hpp
*/
if (num_actions >= MAX_ABSTRACT_ACTIONS) {
error = true;
break;
}
actions[num_actions] = action;
++num_actions;
}
}
actions.resize(num_actions);
//std::cout << "actions: \n";
//for(unsigned i = 0; i < actions.size(); ++i)
//std::cout << ActionsStr[actions[i].type] << " ";
//std::cout << "\n";
/* If you hit this assert, there are too many abstract actions allowed.
* Either coarsen the betting abstraction or increase MAX_ABSTRACT_ACTIONS
* in constants.hpp
*/
assert( !error );
return actions;
}
action_c PotRelationActionAbstraction::get_actions(const Game *game,
const State &state) {
action_c actions(MAX_ABSTRACT_ACTIONS);
int num_actions = 0;
bool error = false;
for (int a = 0; a < NUM_ACTION_TYPES; ++a) {
Action action;
action.type = (ActionType)a;
action.size = 0;
if (action.type == a_raise) {
int32_t min_raise_size;
int32_t max_raise_size;
int32_t pot_size = state.spent[0] + state.spent[1];
if (raiseIsValid(game, &state, &min_raise_size, &max_raise_size)) {
for (int s = 0; s < fractions.size(); ++s) {
int32_t raise_size = state.maxSpent + fractions[s] * pot_size;
if (raise_size > max_raise_size)
raise_size = max_raise_size;
if (raise_size < min_raise_size)
raise_size = min_raise_size;
assert(raise_size > 0);
actions[num_actions] = action;
actions[num_actions].size = raise_size;
++num_actions;
if (raise_size >= max_raise_size)
break;
}
// allin
}
} else if (isValidAction(game, &state, 0, &action)) {
/* If you hit this assert, there are too many abstract actions allowed.
* Either coarsen the betting abstraction or increase
* MAX_ABSTRACT_ACTIONS
* in constants.hpp
*/
if (num_actions >= MAX_ABSTRACT_ACTIONS) {
error = true;
break;
}
actions[num_actions] = action;
++num_actions;
}
}
actions.resize(num_actions);
assert(!error);
return actions;
}
int FcpaActionAbstraction::get_actions( const Game *game,
const State &state,
Action actions
[ MAX_ABSTRACT_ACTIONS ] ) const
{
assert( MAX_ABSTRACT_ACTIONS >= 4 );
int num_actions = 0;
for( int a = 0; a < NUM_ACTION_TYPES; ++a ) {
Action action;
action.type = ( ActionType ) a;
action.size = 0;
if( action.type == a_raise ) {
int32_t min_raise_size;
int32_t max_raise_size;
if( raiseIsValid( game, &state, &min_raise_size, &max_raise_size ) ) {
/* Check for pot-size raise being valid. First, get the pot size. */
int32_t pot = 0;
for( int p = 0; p < game->numPlayers; ++p ) {
pot += state.spent[ p ];
}
/* Add amount needed to call. This gives the size of a pot-sized raise */
uint8_t player = currentPlayer( game, &state );
int amount_to_call = state.maxSpent - state.spent[ player ];
pot += amount_to_call;
/* Raise size is total amount of chips committed over all rounds
* after making the raise.
*/
int pot_raise_size = pot + ( state.spent[ player ] + amount_to_call );
if( pot_raise_size < max_raise_size ) {
actions[ num_actions ] = action;
actions[ num_actions ].size = pot_raise_size;
++num_actions;
}
/* Now add all-in */
actions[ num_actions ] = action;
actions[ num_actions ].size = max_raise_size;
++num_actions;
}
} else if( isValidAction( game, &state, 0, &action ) ) {
/* Fold and call */
actions[ num_actions ] = action;
++num_actions;
}
}
return num_actions;
}
void CrossRiverAction::printSelf() const
{
if(isValidAction())
{
std::cout << "\t||" << std::endl;
std::cout << "(" << m_monkCount << ", ";
std::cout << m_monsterCount << ", ";
std::cout << s_boatLocation[m_to] << ")" << std::endl;
std::cout << "\t||" << std::endl;
std::cout << "\t\\/" << std::endl;
}
}
int NullActionAbstraction::get_actions( const Game *game,
const State &state,
Action actions
[ MAX_ABSTRACT_ACTIONS ] ) const
{
int num_actions = 0;
bool error = false;
for( int a = 0; a < NUM_ACTION_TYPES; ++a ) {
Action action;
action.type = ( ActionType ) a;
action.size = 0;
if( action.type == a_raise ) {
int32_t min_raise_size;
int32_t max_raise_size;
if( raiseIsValid( game, &state, &min_raise_size, &max_raise_size ) ) {
if( num_actions + ( max_raise_size - min_raise_size + 1 )
> MAX_ABSTRACT_ACTIONS ) {
error = true;
break;
}
for( int s = min_raise_size; s <= max_raise_size; ++s ) {
actions[ num_actions ] = action;
actions[ num_actions ].size = s;
++num_actions;
}
}
} else if( isValidAction( game, &state, 0, &action ) ) {
/* If you hit this assert, there are too many abstract actions allowed.
* Either coarsen the betting abstraction or increase MAX_ABSTRACT_ACTIONS
* in constants.hpp
*/
if( num_actions >= MAX_ABSTRACT_ACTIONS ) {
error = true;
break;
}
actions[ num_actions ] = action;
++num_actions;
}
}
/* If you hit this assert, there are too many abstract actions allowed.
* Either coarsen the betting abstraction or increase MAX_ABSTRACT_ACTIONS
* in constants.hpp
*/
assert( !error );
return num_actions;
}
int main( int argc, char **argv )
{
int sock, len, r;
int32_t min, max;
uint16_t port;
Game *game;
MatchState state;
Action action;
struct sockaddr_in addr;
struct hostent *hostent;
FILE *file, *toServer, *fromServer;
char line[ MAX_LINE_LEN ];
if( argc < 4 ) {
fprintf( stderr, "usage: player game server port\n" );
exit( EXIT_FAILURE );
}
file = fopen( argv[ 1 ], "r" );
if( file == NULL ) {
fprintf( stderr, "ERROR: could not open game %s\n", argv[ 1 ] );
exit( EXIT_FAILURE );
}
game = readGame( file );
if( game == NULL ) {
fprintf( stderr, "ERROR: could not read game %s\n", argv[ 1 ] );
exit( EXIT_FAILURE );
}
fclose( file );
hostent = gethostbyname( argv[ 2 ] );
if( hostent == NULL ) {
fprintf( stderr, "ERROR: could not look up address for %s\n", argv[ 2 ] );
exit( EXIT_FAILURE );
}
if( sscanf( argv[ 3 ], "%"SCNu16, &port ) < 1 ) {
fprintf( stderr, "ERROR: invalid port %s\n", argv[ 3 ] );
exit( EXIT_FAILURE );
}
if( ( sock = socket( AF_INET, SOCK_STREAM, 0 ) ) < 0 ) {
fprintf( stderr, "ERROR: could not open socket\n" );
exit( EXIT_FAILURE );
}
addr.sin_family = AF_INET;
addr.sin_port = htons( port );
memcpy( &addr.sin_addr, hostent->h_addr_list[ 0 ], hostent->h_length );
if( connect( sock, (struct sockaddr *)&addr, sizeof( addr ) ) < 0 ) {
fprintf( stderr, "ERROR: could not open connect to %s:%"PRIu16"\n",
argv[ 2 ], port );
exit( EXIT_FAILURE );
}
toServer = fdopen( sock, "w" );
fromServer = fdopen( sock, "r" );
if( toServer == NULL || fromServer == NULL ) {
fprintf( stderr, "ERROR: could not get socket streams\n" );
exit( EXIT_FAILURE );
}
if( fprintf( toServer, "VERSION:%"PRIu32".%"PRIu32".%"PRIu32"\n",
VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION ) != 14 ) {
fprintf( stderr, "ERROR: could not get send version to server\n" );
exit( EXIT_FAILURE );
}
fflush( toServer );
while( fgets( line, MAX_LINE_LEN, fromServer ) ) {
/* ignore comments */
if( line[ 0 ] == '#' || line[ 0 ] == ';' ) {
continue;
}
len = readMatchState( line, game, &state );
if( len < 0 ) {
fprintf( stderr, "ERROR: could not read state %s", line );
exit( EXIT_FAILURE );
}
if( stateFinished( &state.state ) ) {
/* ignore the game over message */
continue;
}
if( currentPlayer( game, &state.state ) != state.viewingPlayer ) {
/* we're not acting */
continue;
}
int bucket = computeHandValue(game,&state.state,state.viewingPlayer,min,max);
/* add a colon (guaranteed to fit because we read a new-line in fgets) */
line[ len ] = ':';
++len;
//if( ( random() % 2 ) && raiseIsValid( game, &state.state, &min, &max ) ) {
/* raise */
if ((bucket >= 3) && raiseIsValid(game, &state.state, &min, &max)) {
action.type = raise;
action.size = min + random() % ( max - min + 1 );
} else if (bucket >=2) {
/* call */
action.type = call;
action.size = 0;
}
else{
action.type = fold;
action.size = 0;
}
if( !isValidAction( game, &state.state, 0, &action ) ) {
fprintf( stderr, "ERROR: chose an invalid action\n" );
exit( EXIT_FAILURE );
}
r = printAction( game, &action, MAX_LINE_LEN - len - 1,
&line[ len ] );
if( r < 0 ) {
fprintf( stderr, "ERROR: line too long after printing action\n" );
exit( EXIT_FAILURE );
}
len += r;
line[ len ] = '\n';
++len;
if( fwrite( line, 1, len, toServer ) != len ) {
fprintf( stderr, "ERROR: could not get send response to server\n" );
exit( EXIT_FAILURE );
}
fflush( toServer );
}
return EXIT_SUCCESS;
}
/* returns >= 0 if match should continue, -1 for failure */
static int processTransactionFile( const Game *game, const int fixedSeats,
uint32_t *handId, uint8_t *player0Seat,
rng_state_t *rng, ErrorInfo *errorInfo,
double totalValue[ MAX_PLAYERS ],
MatchState *state, FILE *file )
{
int c, r;
uint32_t h;
uint8_t s;
Action action;
struct timeval sendTime, recvTime;
char line[ MAX_LINE_LEN ];
while( fgets( line, MAX_LINE_LEN, file ) ) {
/* get the log entry */
/* ACTION */
c = readAction( line, game, &action );
if( c < 0 ) {
fprintf( stderr, "ERROR: could not parse transaction action %s", line );
return -1;
}
/* ACTION HANDID SEND RECV */
if( sscanf( &line[ c ], " %"SCNu32" %zu.%06zu %zu.%06zu%n", &h,
&sendTime.tv_sec, &sendTime.tv_usec,
&recvTime.tv_sec, &recvTime.tv_usec, &r ) < 4 ) {
fprintf( stderr, "ERROR: could not parse transaction stamp %s", line );
return -1;
}
c += r;
/* check that we're processing the expected handId */
if( h != *handId ) {
fprintf( stderr, "ERROR: handId mismatch in transaction log: %s", line );
return -1;
}
/* make sure the action is valid */
if( !isValidAction( game, &state->state, 0, &action ) ) {
fprintf( stderr, "ERROR: invalid action in transaction log: %s", line );
return -1;
}
/* check for any timeout issues */
s = playerToSeat( game, *player0Seat,
currentPlayer( game, &state->state ) );
if( checkErrorTimes( s, &sendTime, &recvTime, errorInfo ) < 0 ) {
fprintf( stderr,
"ERROR: seat %"PRIu8" ran out of time in transaction file\n",
s + 1 );
return -1;
}
doAction( game, &action, &state->state );
if( stateFinished( &state->state ) ) {
/* hand is finished */
/* update the total value for each player */
for( s = 0; s < game->numPlayers; ++s ) {
totalValue[ s ]
+= valueOfState( game, &state->state,
seatToPlayer( game, *player0Seat, s ) );
}
/* move on to next hand */
if( setUpNewHand( game, fixedSeats, handId, player0Seat,
rng, errorInfo, &state->state ) < 0 ) {
return -1;
}
}
}
return 0;
}
/* returns >= 0 if action/size has been set to a valid action
returns -1 for failure (disconnect, timeout, too many bad actions, etc) */
static int readPlayerResponse( const Game *game,
const MatchState *state,
const int quiet,
const uint8_t seat,
const struct timeval *sendTime,
ErrorInfo *errorInfo,
ReadBuf *readBuf,
Action *action,
struct timeval *recvTime )
{
int c, r;
MatchState tempState;
char line[ MAX_LINE_LEN ];
while( 1 ) {
/* read a line of input from player */
struct timeval start;
gettimeofday( &start, NULL );
if( getLine( readBuf, MAX_LINE_LEN, line,
errorInfo->maxResponseMicros ) <= 0 ) {
/* couldn't get any input from player */
struct timeval after;
gettimeofday( &after, NULL );
uint64_t micros_spent =
(uint64_t)( after.tv_sec - start.tv_sec ) * 1000000
+ ( after.tv_usec - start.tv_usec );
fprintf( stderr, "ERROR: could not get action from seat %"PRIu8"\n",
seat + 1 );
// Print out how much time has passed so we can see if this was a
// timeout as opposed to some other sort of failure (e.g., socket
// closing).
fprintf( stderr, "%.1f seconds spent waiting; timeout %.1f\n",
micros_spent / 1000000.0,
errorInfo->maxResponseMicros / 1000000.0);
return -1;
}
/* note when the message arrived */
gettimeofday( recvTime, NULL );
/* log the response */
if( !quiet ) {
fprintf( stderr, "FROM %d at %zu.%06zu %s", seat + 1,
recvTime->tv_sec, recvTime->tv_usec, line );
}
/* ignore comments */
if( line[ 0 ] == '#' || line[ 0 ] == ';' ) {
continue;
}
/* check for any timeout issues */
if( checkErrorTimes( seat, sendTime, recvTime, errorInfo ) < 0 ) {
fprintf( stderr, "ERROR: seat %"PRIu8" ran out of time\n", seat + 1 );
return -1;
}
/* parse out the state */
c = readMatchState( line, game, &tempState );
if( c < 0 ) {
/* couldn't get an intelligible state */
fprintf( stderr, "WARNING: bad state format in response\n" );
continue;
}
/* ignore responses that don't match the current state */
if( !matchStatesEqual( game, state, &tempState ) ) {
fprintf( stderr, "WARNING: ignoring un-requested response\n" );
continue;
}
/* get the action */
if( line[ c++ ] != ':'
|| ( r = readAction( &line[ c ], game, action ) ) < 0 ) {
if( checkErrorInvalidAction( seat, errorInfo ) < 0 ) {
fprintf( stderr, "ERROR: bad action format in response\n" );
}
fprintf( stderr,
"WARNING: bad action format in response, changed to call\n" );
action->type = a_call;
action->size = 0;
goto doneRead;
}
c += r;
/* make sure the action is valid */
if( !isValidAction( game, &state->state, 1, action ) ) {
if( checkErrorInvalidAction( seat, errorInfo ) < 0 ) {
fprintf( stderr, "ERROR: invalid action\n" );
return -1;
}
fprintf( stderr, "WARNING: invalid action, changed to call\n" );
action->type = a_call;
action->size = 0;
}
goto doneRead;
}
doneRead:
return 0;
}
action_c PotRelationActionAbstraction::get_actions(const Game *game,
const State &state){
action_c actions(MAX_ABSTRACT_ACTIONS);
int num_actions = 0;
bool error = false;
for (int a = 0; a < NUM_ACTION_TYPES; ++a) {
Action action;
action.type = (ActionType)a;
action.size = 0;
if (action.type == a_raise) {
int32_t min_raise_size;
int32_t max_raise_size;
int32_t pot_size = state.spent[0] + state.spent[1];
if (raiseIsValid(game, &state, &min_raise_size, &max_raise_size)) {
//std::cout << "min_raise: " << min_raise_size << "\tmax_size: " << max_raise_size << "\tmax_spent:" << state.maxSpent<< "\n";
//std::cout << "min: " << min_raise_size << ", max: " << max_raise_size << ", pot: " << pot_size << ",max spent: " << state.maxSpent << "\n";
for (int s = 0; s < fractions.size(); ++s) {
int32_t raise_size = state.maxSpent + fractions[s] * pot_size;
if( raise_size > max_raise_size )
raise_size = max_raise_size;
if( raise_size < min_raise_size )
raise_size = min_raise_size;
//std::cout << "raise idx: " << s << "\t size: " << raise_size << "\n";
//std::cout << "attempting to raise: " << raise_size << "\n";
assert(raise_size > 0);
actions[num_actions] = action;
actions[num_actions].size = raise_size;
++num_actions;
if( raise_size >= max_raise_size )
break;
}
// allin
//actions[num_actions] = action;
//actions[num_actions].size =
//++num_actions;
}
} else if (isValidAction(game, &state, 0, &action)) {
/* If you hit this assert, there are too many abstract actions allowed.
* Either coarsen the betting abstraction or increase
* MAX_ABSTRACT_ACTIONS
* in constants.hpp
*/
if (num_actions >= MAX_ABSTRACT_ACTIONS) {
error = true;
break;
}
actions[num_actions] = action;
++num_actions;
}
}
//std::cout << "actions: " << num_actions << "\n";
actions.resize(num_actions);
assert( !error );
//for(unsigned i = 0; i < actions.size(); ++i){
//std::cout << i << "\n";
//if(actions[i].type == a_raise)
//assert(actions[i].size > 0);
//}
return actions;
}
/* returns >= 0 if action/size has been set to a valid action
returns -1 for failure (disconnect, timeout, too many bad actions, etc) */
static int readPlayerResponse( const Game *game, const MatchState *state,
const int quiet, const uint8_t seat,
const struct timeval *sendTime,
ErrorInfo *errorInfo,
int seatFD, ReadBuf *readBuf,
Action *action, struct timeval *recvTime )
{
int c, r;
MatchState tempState;
char line[ MAX_LINE_LEN ];
while( 1 ) {
/* read a line of input from player */
if( getLine( seatFD, readBuf, MAX_LINE_LEN, line,
errorInfo->maxResponseMicros ) <= 0 ) {
/* couldn't get any input from player */
fprintf( stderr, "ERROR: could not get action from seat %"PRIu8"\n",
seat + 1 );
return -1;
}
/* note when the message arrived */
gettimeofday( recvTime, NULL );
/* log the response */
if( !quiet ) {
fprintf( stderr, "FROM %d at %zu.%06zu %s", seat + 1,
recvTime->tv_sec, recvTime->tv_usec, line );
}
/* ignore comments */
if( line[ 0 ] == '#' || line[ 0 ] == ';' ) {
continue;
}
/* check for any timeout issues */
if( checkErrorTimes( seat, sendTime, recvTime, errorInfo ) < 0 ) {
fprintf( stderr, "ERROR: seat %"PRIu8" ran out of time\n", seat + 1 );
return -1;
}
/* parse out the state */
c = readMatchState( line, game, &tempState );
if( c < 0 ) {
/* couldn't get an intelligible state */
fprintf( stderr, "WARNING: bad state format in response\n" );
continue;
}
/* ignore responses that don't match the current state */
if( !matchStatesEqual( game, state, &tempState ) ) {
fprintf( stderr, "WARNING: ignoring un-requested response\n" );
continue;
}
/* get the action */
if( line[ c++ ] != ':'
|| ( r = readAction( &line[ c ], game, action ) ) < 0 ) {
if( checkErrorInvalidAction( seat, errorInfo ) < 0 ) {
fprintf( stderr, "ERROR: bad action format in response\n" );
}
fprintf( stderr,
"WARNING: bad action format in response, changed to call\n" );
action->type = call;
action->size = 0;
goto doneRead;
}
c += r;
/* make sure the action is valid */
if( !isValidAction( game, &state->state, 1, action ) ) {
if( checkErrorInvalidAction( seat, errorInfo ) < 0 ) {
fprintf( stderr, "ERROR: invalid action\n" );
return -1;
}
fprintf( stderr, "WARNING: invalid action, changed to call\n" );
action->type = call;
action->size = 0;
}
goto doneRead;
}
doneRead:
return 0;
}
