diff -r 3e7fc096fe5b roundup/cgi/client.py --- a/roundup/cgi/client.py Sat Jul 03 13:04:46 2021 -0400 +++ b/roundup/cgi/client.py Sun Jul 04 21:49:16 2021 -0400 @@ -50,7 +50,7 @@ from email.mime.multipart import MIMEMultipart import roundup.anypy.email_ -from roundup.anypy.strings import s2b, b2s, uchr, is_us +from roundup.anypy.strings import s2b, b2s, bs2b, uchr, is_us def initialiseSecurity(security): '''Create some Permissions and Roles on the security object @@ -2023,13 +2023,134 @@ # most likely case. pass + compressors = [] + try: + import gzip + compressors.append('gzip') + except ImportError: + pass + try: + import brotli + compressors.append('br') + except ImportError: + pass + try: + import zstd + compressors.append('zstd') + except ImportError: + pass + + precompressed_mime_types = [ "image/png", "image/jpeg" ] + + def determine_content_encoding(self, list_all=False): + + # Dumb check. Should parse for q= values and properly order + # the request encodings. Also should handle identity coding. + # Then return first acceptable by q value. + # This code always orders by br, gzip and will send identity + # even if excluded rather than returning 406. + accept_encoding = self.request.headers.get('accept-encoding') or [] + + encoding_list = [] + if ('zstd' in self.compressors) and ('zstd' in accept_encoding): + if not list_all: + return 'zstd' + else: + encoding_list.append('zstd') + if ('br' in self.compressors) and ('br' in accept_encoding): + if not list_all: + return 'br' + else: + encoding_list.append('br') + if ('gzip' in self.compressors) and ('gzip' in accept_encoding): + if not list_all: + return 'gzip' + else: + encoding_list.append('gzip') + + if not list_all: + return None + else: + return encoding_list + + def compress_encode(self, byte_content, quality=4): + encoder = None + # return same content if unable to compress + new_content = byte_content + + # abort if already encoded (e.g. served from + # precompressed file or cache on disk) + if ('Content-Encoding' in self.additional_headers): + return new_content + + # abort if file-type already compressed + if ('Content-Type' in self.additional_headers) and \ + (self.additional_headers['Content-Type'] in \ + self.precompressed_mime_types): + return new_content + + encoder = self.determine_content_encoding() + + if encoder == 'zstd': + new_content = self.zstd.ZSTD_compress(byte_content, 3) + elif encoder == 'br': + # lgblock=0 sets value from quality + new_content = self.brotli.compress(byte_content, + quality=quality, + mode=1, + lgblock=0) + elif encoder == 'gzip': + try: + new_content = self.gzip.compress(byte_content, compresslevel=5) + except AttributeError: + try: + from StringIO import cStringIO as IOBuff + except ImportError: + # python 3 + # however this code should not be needed under python3 + # since py3 gzip library has compress() method. + from io import BytesIO as IOBuff + + out = IOBuff() + # handle under python2 + f = self.gzip.GzipFile(fileobj=out, mode='w', compresslevel=5) + f.write(byte_content) + f.close() + new_content = out.getvalue() + + if encoder: + # we changed the data, change existing content-length header + # and add Content-Encoding header. + self.additional_headers['Content-Length'] = str(len(new_content)) + self.additional_headers['Content-Encoding'] = encoder + + return new_content + def write(self, content): + if not self.headers_done and self.env['REQUEST_METHOD'] != 'HEAD': + # compress_encode modifies headers, must run before self.header() + content = self.compress_encode(bs2b(content)) + if not self.headers_done: self.header() if self.env['REQUEST_METHOD'] != 'HEAD': self._socket_op(self.request.wfile.write, content) def write_html(self, content): + if sys.version_info[0] > 2: + # An action setting appropriate headers for a non-HTML + # response may return a bytes object directly. + if not isinstance(content, bytes): + content = content.encode(self.charset, 'xmlcharrefreplace') + elif self.charset != self.STORAGE_CHARSET: + # recode output + content = content.decode(self.STORAGE_CHARSET, 'replace') + content = content.encode(self.charset, 'xmlcharrefreplace') + + if self.env['REQUEST_METHOD'] != 'HEAD' and not self.headers_done: + # compress_encode modifies headers, must run before self.header() + content = self.compress_encode(bs2b(content)) + if not self.headers_done: # at this point, we are sure about Content-Type if 'Content-Type' not in self.additional_headers: @@ -2041,16 +2162,6 @@ # client doesn't care about content return - if sys.version_info[0] > 2: - # An action setting appropriate headers for a non-HTML - # response may return a bytes object directly. - if not isinstance(content, bytes): - content = content.encode(self.charset, 'xmlcharrefreplace') - elif self.charset != self.STORAGE_CHARSET: - # recode output - content = content.decode(self.STORAGE_CHARSET, 'replace') - content = content.encode(self.charset, 'xmlcharrefreplace') - # and write self._socket_op(self.request.wfile.write, content) @@ -2225,11 +2336,50 @@ def write_file(self, filename): """Send the contents of 'filename' to the user.""" - # Determine the length of the file. - stat_info = os.stat(filename) - length = stat_info[stat.ST_SIZE] # Assume we will return the entire file. offset = 0 + length = -1 # sentinal + + stat_info = os.stat(filename) + + # Determine if we are sending a range. If so, compress + # on the fly. Otherwise see if we have a suitable + # pre-compressed/encoded file we can send. + # FIXME enable/disable with config setting + # don't do this work if admin doesn't ask for it. + if not self.env.get("HTTP_RANGE"): + # no range, search for file in list ordered + # from best to worst alternative + encoding_list = self.determine_content_encoding(list_all=True) + if encoding_list: + # do we need to search through list? If best is not + # precompressed, on the fly compress with best? + # by searching list we will respond with precompressed + # 2nd best or worse. + for encoder in encoding_list: + try: + trial_filename = '%s.%s'%(filename,encoder) + trial_stat_info = os.stat(trial_filename) + if stat_info[stat.ST_MTIME] > \ + trial_stat_info[stat.ST_MTIME]: + # compressed file is obsolete + # don't use it + logger.warning(self._("Cache failure: " + "compressed file %(compressed)s is " + "older than its source file " + "%(filename)s"%{'filename': filename, + 'compressed': trial_filename})) + + continue + filename = trial_filename + stat_info = trial_stat_info + self.setHeader('Content-Encoding', encoder) + break + except FileNotFoundError: + pass + + length = stat_info[stat.ST_SIZE] + # If the headers have not already been finalized, if not self.headers_done: # RFC 2616 14.19: ETag @@ -2255,8 +2405,6 @@ # # Tell the client how much data we are providing. self.setHeader("Content-Length", str(length)) - # Send the HTTP header. - self.header() # If the client doesn't actually want the body, or if we are # indicating an invalid range. if (self.env['REQUEST_METHOD'] == 'HEAD' @@ -2264,6 +2412,7 @@ return # Use the optimized "sendfile" operation, if possible. if hasattr(self.request, "sendfile"): + self.header() self._socket_op(self.request.sendfile, filename, offset, length) return # Fallback to the "write" operation.