refactor loop for url reading
@@ -56,6 +56,7 @@ public:
_contentbuf = new char[_contentbufsize];
}
}
+ bool reset();
int nextUrl();
int getContent();
char *url() const { return _linebuf; }
@@ -63,6 +64,21 @@ public:
~UrlReader() {}
};
+bool UrlReader::reset()
+{
+ _reader.Reset();
+ // Start reading from offset
+ if (_args._singleQueryFile) {
+ _reader.SetFilePos(_args._queryfileOffset);
+ }
+ if (_restarts == _args._restartLimit) {
+ return false;
+ } else if (_args._restartLimit > 0) {
+ _restarts++;
+ }
+ return true;
+}
+
int UrlReader::nextUrl()
{
char *buf = _linebuf;
@@ -78,41 +94,23 @@ int UrlReader::nextUrl()
_leftOvers = NULL;
return _leftOversLen;
}
- // Read maximum to _queryfileOffsetEnd
+ bool again = true;
+ for (int retry = 0; again && retry < 100; ++retry) {
+ // Read maximum to _queryfileEndOffset
if ( _args._singleQueryFile && _reader.GetFilePos() >= _args._queryfileEndOffset ) {
- _reader.SetFilePos(_args._queryfileOffset);
- if (_restarts == _args._restartLimit) {
- return 0;
- } else if (_args._restartLimit > 0) {
- _restarts++;
- }
+ again = reset();
}
int ll = _reader.ReadLine(buf, buflen);
- while (ll > 0 && _args._usePostMode && buf[0] != '/') {
- ll = _reader.ReadLine(buf, buflen);
- }
- if (ll > 0 && (buf[0] == '/' || !_args._usePostMode)) {
+ if (ll > 0) {
+ if (buf[0] == '/' || !_args._usePostMode) {
return ll;
}
- if (_restarts == _args._restartLimit) {
- return 0;
- } else if (_args._restartLimit > 0) {
- _restarts++;
}
if (ll < 0) {
- _reader.Reset();
- // Start reading from offset
- if (_args._singleQueryFile) {
- _reader.SetFilePos(_args._queryfileOffset);
+ // reached EOF
+ again = reset();
}
}
- ll = _reader.ReadLine(buf, buflen);
- while (ll > 0 && _args._usePostMode && buf[0] != '/') {
- ll = _reader.ReadLine(buf, buflen);
- }
- if (ll > 0 && (buf[0] == '/' || !_args._usePostMode)) {
- return ll;
- }
return 0;
}
