GMSPythonCheck.gms : Consistency check for GMSPython

Description

This model performs simple checks on the GMSPython
distribution shipped with GAMS regarding the number of
distributed files as well as the total disk size.

Contributor: Clemens Westphal, April 2020


Category : GAMS Data Utilities library


Main file : GMSPythonCheck.gms   includes :  GMSPythonCheck.gms

$title 'Consistency check for GMSPython' (GMSPYTHONCHECK,SEQ=140)

$ontext
This model performs simple checks on the GMSPython
distribution shipped with GAMS regarding the number of
distributed files as well as the total disk size.

Contributor: Clemens Westphal, April 2020
$offtext

$dropEnv PYTHONUSERBASE

$onEmbeddedCode Python:
import sys
import os
def calcSize(path):
    total = 0
    for r, d, files in os.walk(path):
        if '__pycache__' in d:
            d.remove('__pycache__')
        for f in files:
            f = os.path.join(r, f)
            if not os.path.islink(f): # skip symlinks since os.path.getsize() returns the size of the file behind the link
                total += os.path.getsize(f)
    return total
$offEmbeddedCode

$onEmbeddedCode Python:
platform = sys.platform
gmsPyDir = r'%gams.sysdir%GMSPython'
errors = []
if os.path.isdir(gmsPyDir):
    if platform == 'win32':
        expectedFiles = 927
        expectedSize = 33000000
    elif platform == 'linux':
        expectedFiles = 1034
        expectedSize = 69000000
    elif platform == 'darwin':
        expectedFiles = 1038
        expectedSize = 41024153
    
    SizeLB = expectedSize*0.8
    SizeUB = expectedSize*1.2
    
    files = []
    for r, d, f in os.walk(gmsPyDir):
        if '__pycache__' in d:
            d.remove('__pycache__')
        files.append(f)
    files = [f for l in files for f in l]
    
    numberFiles = len(files)
    size = calcSize(gmsPyDir)
    
    if numberFiles != expectedFiles:
        errors.append("Expected number of files in GMSPython to be " + str(expectedFiles) + " but found " + str(numberFiles))
    if size < SizeLB or size > SizeUB:
        errors.append("Expected size of GMSPython to be between " + str(SizeLB) + " and " + str(SizeUB) + " but got " + str(size))

thirdPartyDir = os.path.join(r'%gams.sysdir%apifiles', 'Python', 'thirdparty')
if os.path.isdir(thirdPartyDir):
    if platform == 'win32':
        expectedFiles = 2083
        expectedSize = 85000000
    elif platform == 'linux':
        expectedFiles = 1958
        expectedSize = 99000000
    elif platform == 'darwin':
        expectedFiles = 1953
        expectedSize = 126000000
    
    SizeLB = expectedSize*0.8
    SizeUB = expectedSize*1.2
    
    files = []
    for r, d, f in os.walk(thirdPartyDir):
        if '__pycache__' in d:
            d.remove('__pycache__')
        files.append(f)
    files = [f for l in files for f in l]
    
    numberFiles = len(files)
    size = calcSize(thirdPartyDir)
    
    if numberFiles != expectedFiles:
        errors.append("Expected number of files in 'thirdparty' directory to be " + str(expectedFiles) + " but found " + str(numberFiles))
    if size < SizeLB or size > SizeUB:
        errors.append("Expected size of 'thirdparty' directory to be between " + str(SizeLB) + " and " + str(SizeUB) + " but got " + str(size))
$offEmbeddedCode


$onEmbeddedCode Python:
pyVersionExpected = '3.8.13'
pyVersion = str(sys.version_info.major) + '.' + str(sys.version_info.minor) + '.' + str(sys.version_info.micro)
if pyVersion != pyVersionExpected:
    errors.append("Expected Python version to be '{}', but found '{}'".format(pyVersionExpected, pyVersion))

with open(os.path.join(gmsPyDir, 'requirements.txt')) as f:
    for l in f.read().splitlines():
        m, v = l.split('==') # m=module, v=version
        m = m.lower()
        if m == 'et-xmlfile':
            m = 'et_xmlfile'
        elif m == 'python-dateutil':
            m = 'dateutil'
        elif m == 'pyyaml':
            m = 'yaml'
        elif m == 'sqlalchemy-access':
            m = 'sqlalchemy_access'
        elif m == 'pywin32':
            continue  # should be pywintypes, but that doesn't have __version__
        elif m == 'psycopg2-binary':
            m = 'psycopg2'

        try:
            module = __import__(m)
        except:
            errors.append("Could not import module '{}'".format(m))
        else:
            mod_version = module.__version__
            if m == 'psycopg2': # '2.9.3 (dt dec pq3 ext lo64)' -> '2.9.3'
                mod_version = mod_version.split(' ')[0]
            if v != mod_version:
                errors.append("Expected '{}' version to be '{}', but found '{}'".format(m, v, module.__version__))
$offEmbeddedCode


$onEmbeddedCode Python:
if errors:
    gams.printLog("\nErrors:")
    for e in errors:
        gams.printLog(e)
    raise Exception("Errors have occurred. See the list above.")
$offEmbeddedCode