-
Notifications
You must be signed in to change notification settings - Fork 1
/
read_after_write.c
executable file
·392 lines (390 loc) · 13.8 KB
/
read_after_write.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
/* Copyright (C) 1992-2007 Univeristy of Minnesota, I/O Performance, Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program in a file named 'Copying'; if not, write to
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139.
*/
/* Author:
* Tom Ruwart (tmruwart@ioperformance.com)
* I/O Perofrmance, Inc.
*/
/*
* This file contains the subroutines necessary to perform a read-after-write
* operation across two different machines running xdd.
*/
#include "xdd.h"
#ifdef FD_SETSIZE
#undef FD_SETSIZE
#define FD_SETSIZE 128
#endif
/*----------------------------------------------------------------------*/
/*
* xdd_raw_err(fmt...)
*
* General-purpose error handling routine. Prints the short message
* provided to standard error, along with some boilerplate information
* such as the program name and errno value. Any remaining arguments
* are used in printed message (the usage here takes the same form as
* printf()).
*/
void
xdd_raw_err(char const *fmt, ...)
{
#ifdef WIN32
LPVOID lpMsgBuf;
#endif
fprintf(xgp->errout, fmt);
perror("reason");
return;
#ifdef WIN32 /* Windows environment, actually */
fprintf(xgp->errout, "last error was %d\n", WSAGetLastError());
FormatMessage(
FORMAT_MESSAGE_ALLOCATE_BUFFER |
FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL,
WSAGetLastError(),
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
(LPTSTR) &lpMsgBuf,
0,
NULL);
fprintf(xgp->errout,"Reason: %s",lpMsgBuf);
#endif /* WIN32 */
} /* end of raw_err() */
/*----------------------------------------------------------------------*/
/*
* xdd_raw_setup_reader_socket()
*
*/
int32_t
xdd_raw_setup_reader_socket(ptds_t *p)
{
/* Create the socket and set some options */
if ((p->raw_sd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
xdd_raw_err("error opening socket");
return(FALSE);
}
/* Bind the name to the socket */
(void) memset(&p->raw_sname, 0, sizeof(p->raw_sname));
p->raw_sname.sin_family = AF_INET;
p->raw_sname.sin_addr.s_addr = htonl(p->raw_addr);
p->raw_sname.sin_port = htons(p->raw_port);
p->raw_snamelen = sizeof(p->raw_sname);
if (bind(p->raw_sd, (struct sockaddr *) &p->raw_sname, p->raw_snamelen)) {
xdd_raw_err("error binding name to socket");
return(FALSE);
}
/* Get and display the name (in case "any" was used) */
if (getsockname(p->raw_sd, (struct sockaddr *) &p->raw_sname, &p->raw_snamelen)) {
xdd_raw_err("error getting socket name");
return(FALSE);
}
p->raw_addr = ntohl(p->raw_sname.sin_addr.s_addr);
p->raw_port = ntohs(p->raw_sname.sin_port);
fprintf(stderr,"%s: Reader Hostname is %s\n\tSocket address is 0x%x = %s\n\tPort %d <0x%x>\n",
xgp->progname,
p->raw_myhostname,
p->raw_sname.sin_addr.s_addr,
inet_ntoa(p->raw_sname.sin_addr),
p->raw_sname.sin_port,p->raw_port);
/* All set; prepare to accept connection requests */
if (listen(p->raw_sd, SOMAXCONN)) {
xdd_raw_err("error starting listening on socket");
return(FALSE);
}
return(TRUE);
} /* end of xdd_raw_setup_reader_socket() */
/*----------------------------------------------------------------------*/
/* xdd_raw_sockets_init() - for read-after-write operations
*
* Windows requires the WinSock startup routine to be run
* before running a bunch of socket routines. We encapsulate
* that here in case some other environment needs something similar.
*
* Returns true if startup was successful, false otherwise.
*
* The sample code I based this on happened to be requesting
* (and verifying) a WinSock DLL version 2.2 environment was
* present, and it worked, so I kept it that way.
*/
int32_t
xdd_raw_sockets_init(void)
{
#ifdef WIN32
WSADATA wsaData; /* Data structure used by WSAStartup */
int wsastatus; /* status returned by WSAStartup */
char *reason;
wsastatus = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (wsastatus != 0) { /* Error in starting the network */
switch (wsastatus) {
case WSASYSNOTREADY:
reason = "Network is down";
break;
case WSAVERNOTSUPPORTED:
reason = "Request version of sockets <2.2> is not supported";
break;
case WSAEINPROGRESS:
reason = "Another Windows Sockets operation is in progress";
break;
case WSAEPROCLIM:
reason = "The limit of the number of sockets tasks has been exceeded";
break;
case WSAEFAULT:
reason = "Program error: pointer to wsaData is not valid";
break;
default:
reason = "Unknown error code";
break;
};
fprintf(xgp->errout,"%s: Error initializing network connection\nReason: %s\n",
xgp->progname, reason);
fflush(xgp->errout);
WSACleanup();
return(FALSE);
}
#endif
return(TRUE);
} /* end of xdd_raw_sockets_init() */
/*----------------------------------------------------------------------*/
/*
* xdd_raw_reader_init() - init the reader side of a read-after-write
*
*/
int32_t
xdd_raw_reader_init(ptds_t *p) {
int status; /* status of various function calls */
/* Set the target_options flag that identifies this target as a reader */
p->target_options |= RX_RAW_READER;
/* init the sockets */
status = xdd_raw_sockets_init();
if (status == FALSE) {
xdd_raw_err("couldn't initialize sockets for RAW reader");
return(FALSE);
}
/* Get the name of the machine that we are running on */
status = gethostname(p->raw_myhostname, sizeof(p->raw_myhostname));
if (status != 0 ) {
xdd_raw_err("could not get hostname");
return(FALSE);
}
/* Get the IP address of this host */
p->raw_hostent = gethostbyname(p->raw_myhostname);
if (! p->raw_hostent) {
xdd_raw_err("unable to identify host");
return(FALSE);
}
p->raw_addr = ntohl(((struct in_addr *) p->raw_hostent->h_addr)->s_addr);
/* Set up the server sockets */
status = xdd_raw_setup_reader_socket(p);
if (status == FALSE) {
xdd_raw_err("could not init raw reader socket");
return(FALSE);
}
/* clear out the csd table */
for (p->raw_current_csd = 0; p->raw_current_csd < FD_SETSIZE; p->raw_current_csd++)
p->raw_csd[p->raw_current_csd] = 0;
p->raw_current_csd = p->raw_next_csd = 0;
/* Initialize the socket sets for select() */
FD_ZERO(&p->raw_readset);
FD_SET(p->raw_sd, &p->raw_readset);
p->raw_active = p->raw_readset;
p->raw_current_csd = p->raw_next_csd = 0;
/* Find out how many sockets are in each set (berkely only) */
#if (IRIX || WIN32 || ALTIX)
p->raw_nd = getdtablehi();
#endif
#if (LINUX || OSX)
p->raw_nd = getdtablesize();
#endif
#if (AIX)
p->raw_nd = FD_SETSIZE;
#endif
#if (SOLARIS || HPUX)
p->raw_nd = FD_SETSIZE;
#endif
p->raw_msg_recv = 0;
p->raw_msg_sequence = 0;
return(TRUE);
} /* end of xdd_raw_reader_init() */
/*----------------------------------------------------------------------*/
/*
* xdd_raw_read_wait() - wait for a message from the writer to do something
*
*/
int32_t
xdd_raw_read_wait(ptds_t *p)
{
int status; /* status of send/recv function calls */
int bytes_received;
int bytes_remaining;
#if (IRIX || WIN32 || ALTIX)
p->raw_nd = getdtablehi();
#endif
status = select(p->raw_nd, &p->raw_readset, NULL, NULL, NULL);
/* Handle all the descriptors that are ready */
/* There are two type of sockets: the one sd socket and multiple
* client sockets. We first check to see if the sd is in the readset.
* If so, this means that a client is trying to make a new connection
* in which case we need to issue an accept to establish the connection
* and obtain a new Client Socket Descriptor (csd).
*/
if (FD_ISSET(p->raw_sd, &p->raw_readset)) { /* Process an incoming connection */
p->raw_current_csd = p->raw_next_csd;
p->raw_csd[p->raw_current_csd] = accept(p->raw_sd, NULL, NULL);
FD_SET(p->raw_csd[p->raw_current_csd], &p->raw_active); /* Mark this fd as active */
FD_SET(p->raw_csd[p->raw_current_csd], &p->raw_readset); /* Put in readset so that it gets processed */
/* Find the next available csd close to the beginning of the CSD array */
p->raw_next_csd = 0;
while (p->raw_csd[p->raw_next_csd] != 0) {
p->raw_next_csd++;
if (p->raw_next_csd == FD_SETSIZE) {
p->raw_next_csd = 0;
fprintf(xgp->errout,"%s: xdd_raw_read_wait: no csd entries left\n",xgp->progname);
break;
}
} /* end of WHILE loop that finds the next csd entry */
} /* End of processing an incoming connection */
/* This section will check to see which of the Client Socket Descriptors
* are in the readset. For those csd's that are ready, a recv is issued to
* receive the incoming data. The clock is then read from pclk() and the
* new clock value is sent back to the client.
*/
for (p->raw_current_csd = 0; p->raw_current_csd < FD_SETSIZE; p->raw_current_csd++) {
if (FD_ISSET(p->raw_csd[p->raw_current_csd], &p->raw_readset)) { /* Process this csd */
/* Receive the writer's current location and length.
*
* When the writer closes the socket we get a read
* indication. Treat any short send or receive as
* a failed connection and silently clean up.
*/
bytes_received = 0;
while (bytes_received < sizeof(p->raw_msg)) {
bytes_remaining = sizeof(p->raw_msg) - bytes_received;
status = recv(p->raw_csd[p->raw_current_csd], (char *) &p->raw_msg+bytes_received, bytes_remaining, 0);
if (status < 0) break;
bytes_received += status;
}
p->raw_msg_recv++;
pclk_now(&p->raw_msg.recvtime);
p->raw_msg.recvtime += xgp->gts_delta;
if (status > 0) status = bytes_received;
if (status == sizeof(p->raw_msg)) { /* Successful receive */
if (p->raw_msg.magic != RX_RAW_MAGIC) {
fprintf(stderr,"xdd_raw_read_wait: Bad magic number %08x on recv %d\n",p->raw_msg.magic, p->raw_msg_recv);
}
if (p->raw_msg.recvtime < p->raw_msg.sendtime) {
fprintf(stderr,"xdd_raw_read_wait: msg %d recv time before send time by %llu picoseconds\n",p->raw_msg.sequence,p->raw_msg.sendtime-p->raw_msg.recvtime);
}
return(TRUE);
} /* end of successful recv processing */
else { /* error on the read operation */
/* At this point, a recv returned an error in which case the connection
* was most likely closed and we need to clear out this csd */
/*"Deactivate" the socket. */
FD_CLR(p->raw_csd[p->raw_current_csd], &p->raw_active);
(void) closesocket(p->raw_csd[p->raw_current_csd]);
p->raw_csd[p->raw_current_csd] = 0;
return(FALSE);
/* indicate that the writer is done and the reader should finish too */
}
} /* End of IF stmnt that processes a CSD */
} /* End of FOR loop that processes all CSDs that were ready */
p->raw_readset = p->raw_active; /* Prepare for the next select */
return(TRUE);
} /* end of xdd_raw_read_wait() */
/*----------------------------------------------------------------------*/
/*
* xdd_raw_setup_writer_socket()
*
*/
int32_t
xdd_raw_setup_writer_socket(ptds_t *p)
{
int status; /* status of send/recv function calls */
char optionvalue; /* used to set the socket option */
/* Create the socket and set some options */
p->raw_sd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (p->raw_sd < 0) {
xdd_raw_err("error opening socket for RAW writer");
return(FALSE);
}
/* Set the TCP_NODELAY option so that packets get sent instantly */
optionvalue = 1;
status = setsockopt(p->raw_sd, IPPROTO_TCP, TCP_NODELAY, &optionvalue, sizeof(optionvalue));
if (status != 0) {
xdd_raw_err("error setting TCP_NODELAY for RAW writer");
}
/* Now build the "name" of the server and connect to it. */
(void) memset(&p->raw_sname, 0, sizeof(p->raw_sname));
p->raw_sname.sin_family = AF_INET;
p->raw_sname.sin_addr.s_addr = htonl(p->raw_addr);
p->raw_sname.sin_port = htons(p->raw_port);
status = connect(p->raw_sd, (struct sockaddr *) &p->raw_sname, sizeof(p->raw_sname));
if (status) {
xdd_raw_err("error connecting to socket for RAW writer");
return(FALSE);
}
return(TRUE);
} /* end of xdd_raw_setup_writer_socket() */
/*----------------------------------------------------------------------*/
/*
* xdd_raw_writer_init() - init the writer side of a read-after-write
*
*/
int32_t
xdd_raw_writer_init(ptds_t *p) {
int status; /* status of various function calls */
/* Set the target_options flag that identifies this target as a writer */
p->target_options |= RX_RAW_WRITER;
/* init the sockets */
status = xdd_raw_sockets_init();
if (status == FALSE) {
xdd_raw_err("couldn't initialize sockets for RAW writer");
return(FALSE);
}
/* Get the IP address of the reader host */
p->raw_hostent = gethostbyname(p->raw_hostname);
if (! p->raw_hostent) {
xdd_raw_err("unable to identify the reader host");
return(FALSE);
}
p->raw_addr = ntohl(((struct in_addr *) p->raw_hostent->h_addr)->s_addr);
/* Set up the server sockets */
status = xdd_raw_setup_writer_socket(p);
if (status == FALSE){
xdd_raw_err("couldn't setup sockets for RAW writer");
return(FALSE);
}
p->raw_msg_sent = 0;
p->raw_msg_sequence = 0;
return(TRUE);
} /* end of xdd_raw_writer_init() */
/*----------------------------------------------------------------------*/
/*
* xdd_raw_writer_send_msg() - send a message from fromt he writer to
* the reader for a read-after-write to be performed.
*/
int32_t
xdd_raw_writer_send_msg(ptds_t *p) {
int status; /* status of various function calls */
pclk_now(&p->raw_msg.sendtime);
p->raw_msg.sendtime += xgp->gts_delta;
status = send(p->raw_sd, (char *) &p->raw_msg, sizeof(p->raw_msg),0);
if (status != sizeof(p->raw_msg)) {
xdd_raw_err("could not send message from RAW writer");
return(FALSE);
}
p->raw_msg_sent++;
p->raw_msg.sequence++;
return(TRUE);
} /* end of xdd_raw_writer_send_msg() */