Remote End Closed Connection without Response #60
Description
SWL had all reports fail on 09/11/2025. It was determined that SWT reports would still run.
They were running repgen5 5.1.6 and 5.2.0
Attempting to curl the URL for T7 and national works.
Fetching: https://cwms-data.usace.army.mil:443/cwms-data/timeseries?name=Bull_Shoals_Dam.Flow-Res+Out.Ave.~1Day.1Day.Regi-Comp&unit=cfs&begin=2025-09-10T15%3A03%3A00&end=2025-09-11T00%3A00%3A00&office=SWL&timezone=US%2FCentral&pageSize=-1
Error fetching: Remote end closed connection without response
Reconnecting to server and trying again
I was able to temporarily fix the issue by commenting out the following lines and replacing them with requests.
This is not a permanent fix and does not make use of maintained sessions among other things.
Suggest we decide if we really don't want to use #24 or if a rewrite of our own source to requests would be better.
Inside value.py I did the following, temporarily, until we decide on a fix!
data = None
retry_until_alternate = 3
while retry_until_alternate > 0:
retry_until_alternate -= 1
if path is None:
path = ""
query = f"/{path}/timeseries?"
# The http(s) guess isn't perfect, but it's good enough. It's for display purposes only.
print("Fetching: %s" % ("https://" if host[-2:] == "43" else "http://") + host+query+params, file=sys.stderr)
r1 = None
try:
if sys.platform != "win32" and self.timeout:
# The SSL handshake can sometimes fail and hang indefinitely
# inflate the timeout slightly, so the socket has a chance to return a timeout error
# This is a failsafe to prevent a hung process
signal.alarm(int(self.timeout * 1.1) + 1)
# print("REQUEST", query+params)
import requests
# NOTE: 09/11/2025 - Eric/Chris called Charles because they got an error about connection closed on the requests
# It was determined that these lines of SSL were causing it by simply switching to requests (which probably uses urllib3) the queries would work
# An issue was reported here for this to bring awareness to the enterprise for a more permananet solution
r1 = requests.get("https://" + host + query+params)
# data = r1.json()
# print(r.json())
# sys.exit()
# if Value._conn is None:
# try:
# from repgen.util.urllib2_tls import TLS1Connection
# Value._conn = TLS1Connection( host, timeout=self.timeout, context=ssl_ctx )
# Value._conn.request("GET", "/{path}" )
# except SSLError as err:
# print(type(err).__name__ + " : " + str(err), file=sys.stderr)
# print("Falling back to non-SSL", file=sys.stderr)
# # SSL not supported (could be standalone instance)
# Value._conn = httplib.HTTPConnection( host, timeout=self.timeout )
# Value._conn.request("GET", "/{path}" )
# # Test if the connection is valid
# Value._conn.getresponse().read()
# Value._conn.request("GET", query+params, None, headers )
# r1 = Value._conn.getresponse()
# getresponse can also hang sometimes, so keep alarm active until after we fetch the response
if sys.platform != "win32" and self.timeout:
signal.alarm(0) # disable the alarm
# Grab the charset from the headers, and decode the response using that if set
# HTTP default charset is iso-8859-1 for text (RFC 2616), and utf-8 for JSON (RFC 4627)
# parts = r1.getheader("Content-Type").split(";")
# charset = "iso-8859-1" if parts[0].startswith("text") else "utf-8" # Default charset
# if len(parts) > 1:
# for prop in parts:
# prop_parts = prop.split("=")
# if len(prop_parts) > 1 and prop_parts[0].lower() == "charset":
# charset = prop_parts[1]
# data = r1.read().decode(charset)
if r1.status_code == 200:
break
print("HTTP Error " + str(r1.status_code) + ": " + data, file=sys.stderr)
if r1.status_code == 404:
r1.json()
# We don't care about the actual error, just if it's valid JSON
# Valid JSON means it was a CDA response, so we treat it as a valid response, and won't retry.
break
except (httplib.NotConnected, httplib.ImproperConnectionState, httplib.BadStatusLine, ValueError, OSError) as e:
print(f"Error fetching: {e}", file=sys.stderr)
if retry_until_alternate == 0 and self.althost is not None and host != self.althost:
print("Trying alternate server", file=sys.stderr)
Value.shared["use_alternate"] = True
(host, path) = (self.althost, self.altpath)
Value._conn = None
retry_until_alternate = 3
else:
print("Reconnecting to server and trying again", file=sys.stderr)
ttime.sleep(3)
try:
Value._conn.close()
except:
pass
Value._conn = None
continue
data_dict = None
try:
data_dict = r1.json()
except json.JSONDecodeError as err:
print(str(err), file=sys.stderr)
print(repr(data), file=sys.stderr)
# get the depth
prev_t = 0
#print repr(data_dict)
if data_dict.get("total", 0) > 0:
for d in data_dict["values"]:
_t = float(d[0])/1000.0 # json returns times in javascript time, milliseconds since epoch, convert to unix time of seconds since epoch
_dt = datetime.datetime.fromtimestamp(_t,pytz.utc)
_dt = _dt.astimezone(self.dbtz)
#_dt = _dt.replace(tzinfo=self.tz)
#print("_dt: %s" % repr(_dt))
#print _dt
if d[1] is not None:
#print("Reading value: %s" % d[1])
_v = float(d[1]) # does not currently implement text operations
else:
_v = None
_q = int(d[2])
self.values.append( ( _dt,_v,_q ) )
else:
print("No values were fetched.", file=sys.stderr)