PyXMake Developer Guide 1.0
PyXMake
Loading...
Searching...
No Matches
Make.py
1# -*- coding: utf-8 -*-
2# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
3# % Make Module - Classes and Functions %
4# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
5"""
6Create a make object to define the building environment and to execute the
7build commands. The make event is subdivided in a pre-, main- and a post-build
8event.
9
10@note: PyXMake module
11Created on 20.03.2018
12
13@version: 1.0
14----------------------------------------------------------------------------------------------
15@requires:
16 -
17
18@change:
19 -
20
21@author: garb_ma [DLR-FA,STM Braunschweig]
22----------------------------------------------------------------------------------------------
23"""
24
25## @package PyXMake.Build.Make
26# Create a make object to define the building environment.
27## @author
28# Marc Garbade
29## @date
30# 20.03.2018
31## @par Notes/Changes
32# - Added documentation // mg 29.03.2018
33try:
34 from builtins import object
35except ImportError:
36 pass
37
38try:
39 FileNotFoundError
40except NameError:
41 FileNotFoundError = IOError # @ReservedAssignment
42
43import sys, os, platform
44import argparse
45import paramiko
46import abc, six
47import shutil
48import inspect
49import stat
50import json
51import copy
52import shlex
53import ntpath
54import posixpath
55import random, string
56import subprocess
57import multiprocessing
58import tempfile
59import colorsys
60import psutil
61import re
62import io
63import uuid
64import socket
65import getpass
66import base64
67import logging
68import warnings
69import importlib
70
71import numpy as np
72
73from packaging import version
74from shutil import copyfile
75from types import MethodType
76from collections import OrderedDict #@UnresolvedImport
77from PIL import Image, ImageColor
78
79try: # pragma: no cover
80 ## Add additional path to environment variable
81 if os.path.exists(os.path.join(sys.prefix,"conda-meta")) and not os.path.join(sys.prefix,"conda-meta") in os.getenv("PATH",""):
82 os.environ["PATH"] = os.pathsep.join([os.path.join(sys.prefix,"Library","bin"),os.getenv("PATH","")])
83 # Now the requests module can be load w/o errors.
84 import requests
85# Fail gracefully
86except: pass
87
88from ..Tools import Utility
89
90## Absolute system path to PyXMake.
91PyXMakePath = Utility.GetPyXMakePath()
92## Absolute system path to configuration files.
93Path2Config = os.path.join(PyXMakePath,"Build","config")
94# This is the default build option for all classes/methods inherited from this module going forward. Can be overwritten.
95AllowDefaultMakeOption = bool(int("-1" if __debug__ or not getattr(sys,"frozen",False) else int(os.getenv("pyx_default_make_opt","0"))) == -1)
96
97## Create an alias using default logger for all print statements
98logger = logging.getLogger(__name__)
99# setattr(sys.modules[__name__],"print", logger.info)
100
101## @class PyXMake.Build.Make.OS
102# Abstract base class for all system subclasses. Inherited from built-in ABCMeta & object.
103# Compatible with both Python 2.x and 3.x.
105 """
106 Base class of all supported subsystems.
107 """
108 def __init__(self, *args, **kwargs):
109 ## String identifier of current instance.
110 self.SystemObjectKind = "Base"
111 pass
112
113## @class PyXMake.Build.Make.NT
114# Abstract base class for all NT subclasses. Inherited from built-in ABCMeta & object.
115# Compatible with both Python 2.x and 3.x.
116@six.add_metaclass(abc.ABCMeta)
117class NT(OS):
118 """
119 Inherited class to NT projects without any presets.
120 """
121 @abc.abstractmethod
122 def __init__(self, *args, **kwargs):
123 """
124 Initialization of NT class object.
125 """
126 super(NT, self).__init__(*args, **kwargs)
127 ## String identifier of current instance.
129
130## @class PyXMake.Build.Make.POSIX
131# Abstract base class for all POSIX subclasses. Inherited from built-in ABCMeta & object.
132# Compatible with both Python 2.x and 3.x.
133@six.add_metaclass(abc.ABCMeta)
134class POSIX(OS):
135 """
136 Inherited class to POSIX projects without any presets.
137 """
138 @abc.abstractmethod
139 def __init__(self, *args, **kwargs):
140 """
141 Initialization of POSIX class object.
142 """
143 super(POSIX, self).__init__(*args, **kwargs)
144 ## String identifier of current instance.
146 ## Overwrite create method in all subclasses to use a predefined MakeFile for all builds. This implements
147 # the use of gcc, g++ and gfortran as well as Mingw64 and Linux shell support. Convoluted, but working.
148 setattr(self, "create", self.__create____create__)
149 # Copy Makefile to current scratch directory.
150 copyfile(os.path.join(Path2Config,"stm_makefile"), os.path.join(self.scrtdir,"Makefile"))
151 # Add temporary Makefile to tuple scheduled for removal
152 self.temps = self.temps + ("Makefile",)
153
154 def __create__(self, **kwargs): # pragma: no cover
155 """
156 Unified create function replacing all create commands of ALL classes when used with Mingx64 or on Linux. All builds are solely
157 defined by one unified Makefile. in these cases.
158 """
159 # Access some sub-packages
160 from PyXMake.Build import __install__ #@UnresolvedImport
161 from PyXMake.VTL import GetPreprocessingCommand
162 from numpy.distutils.fcompiler import gnu
163 from numpy.distutils import ccompiler
164 from numpy.f2py import crackfortran, auxfuncs
165 from numpy.distutils import misc_util
166 from packaging.version import parse
167
168 # Space delimiter
169 delimn = " ";
170
171 # Create a local copy of the current environment
172 self.environ = copy.deepcopy(getattr(os.environ,"_data",{}))
173
174 # Validate third party dependencies
175 if Utility.GetPlatform() in ["windows"] and not Utility.GetExecutable("choco"): # pragma: no cover
176 os.environ["pyx_user_install"] = "chocolatey"
177 os.system(" ".join([sys.executable,os.path.abspath(__install__.__file__)]))
178
179 ## Add local binary directory to the global path
180 os.environ["PATH"] = os.pathsep.join([os.getenv("PATH"),os.path.join(os.path.dirname(os.path.abspath(__install__.__file__)),"bin","chocolatey")])
181
182 try:
183 # Get base path, get compiler path defaulting to the latest version and get base initialization shell script
184 msys2_base_path, msys2_compiler_path, msys2_shell_initialization = self.setup(mingw = Utility.GetExecutable("choco"))
185 # Update PreProcessing command if default is not available
186 if self.hasFoss and ("fpp" in getattr(self,"precmd","") and not Utility.GetExecutable("fpp")): # pragma: no cover
187 self.precmdprecmd = GetPreprocessingCommand(1).split() + self.precmdprecmd.split()[4:]; self.precmdprecmd.insert(-1,"-o")
188 self.precmdprecmd = delimn.join(self.precmdprecmd)
189 # On NT systems using ming, the quotes have to be changed. Otherwise, a syntax error occurs.
190 if Utility.GetPlatform() in ["windows"]: self.precmdprecmd = self.precmdprecmd.replace('"',"'")
191 # Assemble the final command
192 self.precmdprecmd = delimn.join([msys2_shell_initialization,"-c",'"%s"' % self.precmdprecmd])
193 except:
194 # Only meaningful on windows systems
195 if Utility.GetPlatform() in ["windows"]: raise ImportError
196 # Set a dummy value
197 msys2_base_path = ""
198 msys2_compiler_path = ""
199 msys2_shell_initialization = ""
200
201 try:
202 # Resolving compatibility issue > Python 3.6
203 from re.RegexFlag import MULTILINE
204 except ImportError:
205 # Python 3.5 & lower
206 from re import MULTILINE
207
208 # Go into scratch directory (if defined). This directory has to include a Makefile.
210
211 # Verify that module files are in the same path as any output from f2py
212 if self.MakeObjectKind in ['Py2X',"f2py"]:
213 self.incdirs.append(os.path.join(os.getcwd(),"tmp","Release"))
214
215 # Rewrite all path to be Linux/Mingw64 compliant
216 os.environ["pyx_compiler"] = os.getenv("pyx_compiler").replace(ntpath.sep,posixpath.sep)
217
218 if Utility.GetPlatform() in ["windows"]:
219 # Only add dependencies for MSYS on Windows.
220 os.environ["pyx_ldflags"] = delimn.join(["-L"+os.path.join(msys2_base_path,"mingw64","lib"),"-L"+msys2_compiler_path]) + delimn
221 if parse(np.__version__) >= parse("1.22.0"): os.environ["pyx_ldflags"] = delimn.join([os.getenv("pyx_ldflags",""), "-D__STDC_NO_THREADS__"]) + delimn
222
223 # Add all include paths to the command string
224 for x in ['-I"'+x+'" ' for x in self.incdirs]: os.environ["pyx_ldflags"] = os.getenv("pyx_ldflags","") + x
225
226 # Add all dependency paths to the command string
227 for x in ['-L"'+x+'" ' for x in self.libdirs]: os.environ["pyx_ldflags"] = os.getenv("pyx_ldflags","") + x
228
229 # Add all required libraries to the command string
230 for x in ['-l'+x+' ' for x in self.libs]: os.environ["pyx_ldflags"] = os.getenv("pyx_ldflags","") + x
231
232 # Publish additional user commands as an environment variable
233 os.environ["pyx_ldflags"] = os.getenv("pyx_ldflags","").replace("\\","/")
234
235 # Pre-build event (if required)
236 try:
237 if self.precmdprecmd != '' and self.precmdprecmd.strip() != self.iniCompiler:
238 command = self.precmdprecmd.split(delimn); command.insert(-2,"-D__GFORTRAN__");
239 # Assemble command
240 self.precmdprecmd = delimn.join(command)
241 # Execute the command
242 Utility.Popen(self.precmdprecmd, self.verbose)
243 except: pass
244
245 # Loop over all source files and apply cross-compilation pre-processing to all of them.
246 try:
247 for source in os.getenv("pyx_source").split(delimn):
248 Make.F2CPreprocessing(source)
249 except FileNotFoundError:
250 if os.path.isfile(self.intermediate_wrapper):
251 Utility.ReplaceTextinFile(self.intermediate_wrapper, self.wrapper_module, {'%pyx_source%':'"'+self.buildname+'"'}, source=self.scrtdir)
252 Make.F2CPreprocessing(self.wrapper_module)
253 os.environ["pyx_source"] = self.wrapper_module
254 except UnicodeError: pass
255
256 # Strip decorator commands from the original make command. These are relevant for the Makefile as well.
257 os.environ["pyx_cflags"] = delimn.join([os.getenv("pyx_cflags",""),delimn.join([x for x in self.makecmd.split(delimn) if x.startswith("-D")])])
258
259 ## Base command for all make commands
260 # Windows with MSYS2
261 command = msys2_shell_initialization
262 # Linux
263 if Utility.GetPlatform() != "windows": command = delimn.join([';export PYX_BUILDID="'+os.environ["pyx_buildid"]+'"',';export PYX_SOURCE="'+os.environ["pyx_source"]+'"',";bash"])
264
265 # If used with Python and numpy, check if numpy version is sufficient and patch-able.
266 if self.MakeObjectKind in ['Py2X',"f2py"]: # pragma: no cover
267 # Modify numpy on the fly
268 for x in [ccompiler, crackfortran, gnu, misc_util]:
269 # Exception handling when executing in a Docker container instance.
270 if Utility.IsDockerContainer() and Utility.GetPlatform() in ["linux"] and not os.access(x.__file__, os.W_OK):
271 Permission = str(oct(stat.S_IMODE(os.lstat(x.__file__).st_mode)));
272 subprocess.check_call(["sudo","chmod","777",x.__file__])
273
274 # Cross compilation flags (GNU style format)
275 os.environ["pyx_cflags"] += delimn + "-ffree-line-length-0"
276 if self.makecmd.find("-fixed") != -1:
277 os.environ["pyx_cflags"] += delimn + "-f%s-form" % "fixed"
278 else:
279 os.environ["pyx_cflags"] += delimn + "-f%s-form" % "free"
280
281 # Output detailed information from f2py when increasing the verbose level
282 if self.verbose >= 2: os.environ["pyx_ldflags"] += delimn + "--verbose"
283 else: os.environ["pyx_ldflags"] += delimn + "--quiet"
284
285 # Add additional command line options to the final command
286 if Utility.GetPlatform() != "windows":
287 # Fetch compiler request on Linux. Remove file extension and path from f2py call.
288 command = command.split(delimn); compiler = Utility.PathLeaf(os.getenv("pyx_compiler"))
289 command.insert(0,'export PYX_COMPILER="'+compiler.replace(os.path.splitext(compiler)[1],"")+'"');
290 command.insert(0,'export PYX_CFLAGS="'+os.getenv("pyx_cflags")+'"')
291 command.insert(0,'export PYX_LDFLAGS="'+os.getenv("pyx_ldflags")+'"')
292 command.insert(0,'export LD_RUN_PATH="'+os.pathsep.join(self.libdirs)+'"')
293 command.insert(0,'export NPY_DISTUTILS_APPEND_FLAGS=1')
294 command = delimn.join(command)
295
296 ## This beauty ensures backwards compatibility with older f2py versions. Upon 1.19, there persists a bug in the original implementation,
297 # preventing the correct version to be identified. This bug has since been resolved. If the version is sufficient, simply run everything on a dummy file.
298 with tempfile.NamedTemporaryFile(mode='w+') as __:
299 # Modify f2py's routines to support UTF-8 in any case
300 with open(gnu.__file__,"r") as f: gnustream = f.read()
301 with open(misc_util.__file__,"r") as f: utilstream = f.read();
302 with open(auxfuncs.__file__,"r") as f: auxstream = f.read();
303 with open(crackfortran.__file__,"r") as f: crackstream = f.read();
304 with open(ccompiler.__file__,"r") as f: ccompstream = f.read()
305 # Replace substring if version is not sufficient.
306 with open(gnu.__file__,"w") as f:
307 # Replace the following with numpy's source code. This was fixed in version 1.19.1
308 target = "v >= '4.'"; replace = "int(v.split('.')[0]) >= int('4')";
309 pattern = re.compile(target,MULTILINE); _ = gnustream;
310 # Replace substring if version is not sufficient.
311 if Utility.GetPlatform() in ["windows"]: _ = gnustream.replace('["<F90>", "-Wall", "-g"],','["<F90>", "-Wall", "-g","-static","-static-libgcc","-static-libgfortran"],')
312 if re.search(pattern,_) and not parse(np.__version__) >= parse("1.19.0"):
313 if self.verbose >= 2:
314 print("==================================")
315 print("On-the-fly patching of numpy version %s" % np.__version__)
316 print("==================================")
317 _ = re.sub(pattern,replace,_)
318 f.write(_)
319 # Replace all problematic statements
320 with open(misc_util.__file__,"w") as f:
321 # Modify statements in-place
322 if utilstream.find(str("import textwrap")) != -1:
323 _ = utilstream.replace(str("import textwrap"),str("import textwrap, io"))
324 else: _ = utilstream.replace(str("import os"),str("import os, io"))
325 # Enforce correct encoding.
326 _ = _.replace(str("open(source, 'r')"),str('io.open(source, "r", encoding="utf-8")')); f.write(_)
327 with open(crackfortran.__file__,"w") as f:
328 # Modify statements in-place
329 _ = crackstream.replace(str("import fileinput"),str("import fileinput, io"))
330 _ = _.replace(str("fileinput.FileInput(ffile)"),str("fileinput.FileInput(ffile,openhook=fileinput.hook_encoded('utf-8'))"))
331 # Check if version is below 3.
332 if sys.version_info <= (3,0): _ = _.replace(str(r'\xa0'),str(r'\t'))
333 # Verify that fixed form format is correctly identified under all circumstances
334 if self.makecmd.find("-fixed") != -1 and parse(np.__version__) >= parse("1.26.0"):
335 # This has two hits. But one is in the __main__ section of the code and thus irrelevant
336 _ = _.replace(str("sourcecodeform = 'free'"),str("sourcecodeform = 'fix'"));
337 # Modify source code
338 _ = _.replace(str("open(file, 'r')"),str('io.open(file, "r", encoding="utf-8")')); f.write(_)
339 # Modify auxiliary code when using old version. Established backwards compatibility.
340 if sys.version_info <= (3,0):
341 with open(auxfuncs.__file__,"w") as f:
342 # Ensure that all rules can be found. No type mixing allowed.
343 _ = auxstream.replace(str("if isinstance(rules[k], str)"),str("if isinstance(rules[k], basestring)"))
344 _ = _.replace("ret[k] = replace(rules[k], d)","ret[k] = str(replace(str(rules[k]), d))")
345 f.write(_)
346 ## Always compile all sources with permissive. For now.
347 # Use this hack to sneak the compile option to f2py
348 if parse(np.__version__) < parse("2.0.0"):
349 with open(ccompiler.__file__,"w") as f:
350 # Ensure that source code is compiled with relaxed rules
351 _ = ccompstream.replace("ccomp = self.compiler_so","self.compiler_so += ['-fpermissive','-fno-strict-aliasing']; ccomp = self.compiler_so")
352 f.write(_)
353 ## HOTFIX for older interpreters. Do not mess with the default encoding.
354 # Modify auxiliary code when using old version. Established backwards compatibility.
355 if kwargs.get("use_default_encoding",sys.version_info <= (3,0)):
356 # Modify f2py to support UTF-8 in any case
357 with open(misc_util.__file__,"w") as f: f.write(utilstream)
358 with open(crackfortran.__file__,"w") as f: f.write(crackstream)
359 with open(auxfuncs.__file__,"w") as f: f.write(auxstream)
360 try:
361 # Execute build command
362 params = {} if not Utility.IsDockerContainer() else {"shell":True,"collect":True}
363 # Modify global encoding of the old interpreter
364 environment = os.environ.copy();
365 # Update environment variable
366 if sys.version_info <= (3, 0): params.update({"env":environment})
367 # Do not remove newline from output strings
368 if Utility.GetPlatform() in ["windows"] and not Utility.IsDockerContainer():
369 params.update({"replace":""})
370 command = delimn.join([command,"-c",'"make f2py"'])
371 # Execute the command
372 Utility.Popen(command, verbosity=self.verbose, **params)
373 except:
374 pass
375 finally:
376 # Modify f2py to support UTF-8 in any case
377 with open(gnu.__file__,"w") as f: f.write(gnustream)
378 with open(misc_util.__file__,"w") as f: f.write(utilstream)
379 with open(crackfortran.__file__,"w") as f: f.write(crackstream)
380 with open(ccompiler.__file__,"w") as f: f.write(ccompstream)
381 with open(auxfuncs.__file__,"w") as f: f.write(auxstream)
382 # Print information for the user.
383 if self.verbose >= 2:
384 print("==================================")
385 print("Restoring numpy version %s" % np.__version__)
386 print("==================================")
387 # Delete temporary folders
388 if os.path.exists("tmp") or os.path.exists(os.getenv("pyx_buildid")):
389 try:
390 shutil.rmtree("tmp"); shutil.rmtree(os.getenv("pyx_buildid"))
391 except (FileNotFoundError, OSError) as _: pass
392
393 # Restore default access rights.
394 if Utility.IsDockerContainer() and Utility.GetPlatform in ["linux"]: subprocess.check_call(["sudo","chmod",Permission,gnu.__file__])
395
396 elif self.MakeObjectKind in ['Fortran',"CCxx"]:
397 # Cross compilation flags (GNU style format)
398 if self.makecmd.find("-fixed") != -1:
399 os.environ["pyx_cflags"] += delimn + "-ffixed-line-length-132 -f%s-form" % "fixed"
400 elif self.MakeObjectKind in ['Fortran']:
401 os.environ["pyx_cflags"] += delimn + "-ffree-line-length-0 -f%s-form" % "free"
402
403 # Add additional command line options to the final command
404 if Utility.GetPlatform() != "windows":
405 command = command.split(delimn);
406 command.insert(0,'export PYX_CFLAGS="'+os.getenv("pyx_cflags")+'"')
407 command.insert(0,'export PYX_LDFLAGS="'+os.getenv("pyx_ldflags")+'"')
408 command = delimn.join(command)
409
410 # Execute build command
411 command = delimn.join([command,"-c",'"make"'])
412 Utility.Popen(command, verbosity=self.verbose, replace="")
413
414 # Add temporary files to tuple scheduled for removal
415 self.temps = self.temps + (".o",)
416
417 # Remove module files. We have to process them later.
418 _ = list(self.temps);
419 if ".mod" in _: _.remove(".mod");
420 self.temps = tuple(_)
421
422 # Finish and delete redundant files and folders
423 Utility.DeleteFilesbyEnding(self.temps)
424
425 # Copy all unprocessed files to the output folder
426 for x in os.listdir(os.getcwd()):
427 try:
428 ## Accept both OutLibs and OutDir variable. Checks for existence of OutLibs first.
429 if os.path.isfile(x) and not x.startswith("."): shutil.move(x,os.path.join(getattr(self,"outlibs",self.outdir),x))
430 except: pass
431
432 # Copy module and header files to output module folder (if required)
433 if hasattr(self, "outlibs") and hasattr(self, "outmodule"):
434 if self.outlibs != self.outmodule:
436 # Only create folders if any module or header files actually exists
437 if any([x.endswith((".mod",".h")) for x in os.listdir(os.getcwd())]): os.makedirs(self.outmodule, exist_ok=True)
438 for x in os.listdir(os.getcwd()):
439 if x.endswith((".mod",".h")): shutil.move(x,os.path.join(self.outmodule,x))
440
441 # Remove environment variable in case of multiple builds
442 os.environ.pop("pyx_cflags","")
443 os.environ.pop("pyx_ldflags","")
444
445 # Recreate environment upon job finished
446 for k, v in getattr(self,"environ",{}).items():
447 # Prevent problems when keys or values became encoded
448 if hasattr(k,"decode"): k = k.decode()
449 if hasattr(v,"decode"): v = v.decode()
450 os.environ.update({k:v})
451 pass
452
453## @class PyXMake.Build.Make.Make
454# Abstract base class for all make objects. Inherited from built-in ABCMeta & object.
455# Compatible with both Python 2.x and 3.x.
457 """
458 Parent class for all make objects.
459 """
460 @abc.abstractmethod
461 def __init__(self, BuildID, Srcs, scratch=os.getcwd(), verbose=0, *args, **kwargs):
462 """
463 Low-level initialization of parent class.
464 """
465 ## Base string of build object.
466 # Defines the base string of the current build object. The final build name
467 # used in the instanced objects is assembled using this immutable base id.
468 self.buildid = BuildID
469 ## Source file or folders
470 self.srcs = []
471 self.srcs.append(Srcs)
472 self.srcs = list(Utility.ArbitraryFlattening(self.srcs))
473 # The given class is run bare. Child classes can be programmed to react properly to this situation
474 self.bare = not BuildID and not self.srcs
475 ## Source file type
476 self.stype = kwargs.get("stype",'Fortran')
477 ## Level of verbosity of the current build object.
478 # Define the verbosity level of the current build object. Defaults to 0 and suppresses
479 # all outputs to the command line. A higher value increases the level of verbosity up to
480 # a maximum level of 2.
481 self.verbose = verbose
482 ## Toggle between free open source software and commercial 3rd party libraries. On POSIX systems,
483 # only free open source software is supported. On Windows, the Intel Compiler Library as well as the package manager
484 # MINGW64 are natively supported. All other variants have no supported presets.
485 self.hasFoss = kwargs.get("foss", Utility.GetExecutable("choco") or Utility.GetPlatform() in ["linux"] or kwargs.get("bash",False))
486
487 # Set scratch directory & default input/ output locations
489 ## Current scratch directory
490 self.scrtdir = os.getcwd()
491 ## Default search directory for source files.
492 self.srcdir = os.getcwd()
493 ## Default search directory for output.
494 self.outdir = os.getcwd()
495
496 # Read Intel path from Paths.log. Only if present
497 if os.path.exists(os.path.join(PyXMakePath,'Paths.log')):
498 with open(os.path.join(PyXMakePath,'Paths.log')) as f:
499 content = f.readlines()
500 content = [x.strip() for x in content][20]
501 # Do not use paths provided explicitly
502 else: content = ""
503 ## Path to Intel Fortran Compiler (read from Paths.log or empty).
504 self.intelpath = content
505
506 # Initialization of tuple containing temporary files
507 ## Tuple of data to be removed after job completion.
508 self.temps = ()
509
510 # Initialization of lists containing additional sources, modules or libraries
511 ## List of include directories.
512 self.incdirs = []
513 ## List of library directories.
514 self.libdirs = []
515 ## List of actual libraries (by name) used during linking.
516 self.libs = []
517
518 # Initialization of list containing files to be copied to the output directory.
519 ## List of files to be copied to the output directory after finish.
520 self.copyfiles = []
521
522 ## Default initialization of compiler script.
523 # Only set on NT systems. There is no compiler initialization script for Linux
524 self.iniCompiler = ""
525
526 ## Define the architecture for the build directly by using the keyword argument "arch".
527 # Defaults to None, in which case the architecture is determined by using the python executable.
528 self.setarch = True if kwargs.get('arch', None) in ['x86', 'x64'] else False
529
530 ## Default version of Microsoft visual studio used by the Intel Fortran Compiler. Defaults to 'vs2015'.
531 self.msvsc = kwargs.get("msvsc",'vs2015')
532 os.environ['pyx_msvsc'] = self.msvsc
533
534 # Set variables in dependence of identified system
535 if 'win' in sys.platform:
536
537 if (('32bit' in platform.architecture() and not self.setarch) or (self.setarch and kwargs.get('arch', None) == "x86")):
538 # Set variables for x86
539 os.environ['pyx_proc'] = 'x86'
540 os.environ['pyx_intel'] = 'ia32'
541 ## Processor architecture
542 self.architecture = 'x86'
543
544 elif (('64bit' in platform.architecture() and not self.setarch) or (self.setarch and kwargs.get('arch', None) == "x64")):
545 # Set variables for x64
546 os.environ['pyx_proc'] = 'amd64'
547 os.environ['pyx_intel'] = 'intel64'
548 self.architecture = 'x64'
549
550 ## Executable batch script (including absolute system path) to set up the Intel Fortran Compiler.
551 if kwargs.get("initialize",True):
552 # Set Intel Compiler path for backwards compatibility
553 if not self.intelpath: _, self.intelpath, self.iniCompiler = self.setup(**kwargs)
554 # Modern convention
555 else: self.iniCompiler = self.setup(**kwargs)[-1]
556
557 elif Utility.IsDockerContainer() and Utility.GetPlatform() in ["linux"]:
558 # Set default environment variables
559 os.environ["pyx_intel"] = "intel64_lin"
560 # Distributed Intel runtime in Docker.
561 self.intelpath = os.path.join(Utility.AsDrive("opt"),"intel","psxe_runtime","linux")
562 # There is no difference on linux. This will default to x64 in all cases.
563 self.architecture = Utility.GetArchitecture()
564
565 elif 'linux' in sys.platform:
566 # Set default environment variables
567 os.environ["pyx_intel"] = ""
568 # There is no difference on linux. This will default to x64 in all cases.
569 self.architecture = Utility.GetArchitecture()
570
571 ## Post build command. Defaults to an empty string.
572 self.postcmd = ""
573
574 # Always add MKL library
575 ## Uses absolute system path of MKL library.
576 if os.path.exists(os.path.join(self.intelpath,"mkl","include")):
577 # Additional include source files. They are not part of the redistribution package
578 mkl_include_path = os.path.join(self.intelpath,"mkl","include")
579 self.incdirs.append(mkl_include_path)
580
581 if os.path.exists(os.path.join(self.intelpath,"mkl")):
582 # Path to MKL and compiler library. These are part of any redistribution package.
583 mkl_lib_path = os.path.join(self.intelpath,"mkl","lib",os.environ['pyx_intel'])
584 mkl_compiler_path = os.path.join(self.intelpath,"compiler","lib",os.environ['pyx_intel'])
585 self.libdirs.extend([mkl_lib_path, mkl_compiler_path])
586
587 # Update environment build path
588 if os.path.exists(os.path.join(self.intelpath,"bin",os.environ['pyx_intel'])):
589 os.environ["PATH"] = ";".join([os.getenv("PATH",""),os.path.join(self.intelpath,"bin",os.environ['pyx_intel'])])
590
591 # Collect all MKL include files (if given!)
592 self._mkl_includes = []; mkl_paths = [x for x in self.incdirs if "mkl" in x]
593 for path in mkl_paths: self._mkl_includes.extend([x for x in os.listdir(path) if (os.path.isfile(os.path.join(path,x)) and x.endswith((".f90",".F90",".for",".FOR",".f",".F",".f77",".F77")))])
594
595 # Dynamic second inheritance in dependence of supplied keyword argument POSIX.
596 self.__posix__(**kwargs)
597
598 # Add all source files to list of files to remove from scratch folder. No exceptions.
599 self.temps = self.temps + tuple((os.path.join(self.scrtdir,x) for x in self.srcs))
600 pass
601
602 def __posix__(self,**kwargs):
603 """
604 Request compatibility with Mingw64 and Linux.
605 """
606 # Dynamic second inheritance in dependence of supplied keyword argument POSIX.
607 if kwargs.get("bash", self.hasFoss):
608 try:
609 # Attempt to initialize POSIX support
610 POSIX.__init__(self)
611 except:
612 pass
613 else:
614 try:
615 ## Delete prototype of create function.
616 # Its content was renamed to "create" in POSIX.__init__()
617 delattr(self, "__create__")
618 except:
619 pass
620
621 @staticmethod
622 def __parser__(): # pragma: no cover
623 """
624 Default parser object for command line interface
625
626 @author: Marc Garbade
627 """
628 # Process all known arguments
629 parser = argparse.ArgumentParser(add_help=False)
630 # Default parser object with settings shared by most commands.
631 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
632 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
633 parser.add_argument('-f', '--files', nargs='+', default=[], help="Source file or list of all source files in the order of compilation")
634 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
635 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional directories and files required for the build.")
636 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for the build. Defaults to current workspace.")
637 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
638 # Return default parser object
639 return parser
640
641 @staticmethod
642 def Detach(): # pragma: no cover
643 """
644 Detach current console window from parent window.
645
646 @author: Marc Garbade
647 """
648 kwargs = {}
649 # Set system/version dependent "start_new_session" analogs
650 if os.name == 'nt': # Windows
651 DETACHED_PROCESS = 0x00000008 # 0x8 | 0x200 == 0x208
652 CREATE_NEW_PROCESS_GROUP = subprocess.CREATE_NEW_PROCESS_GROUP
653 kwargs.update(creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
654 elif sys.version_info < (3, 2): # Posix
655 kwargs.update(preexec_fn=os.setsid) # @UndefinedVariable
656 else: # Python 3.2+ and Unix
657 kwargs.update(start_new_session=True)
658
659 return kwargs
660
661 @staticmethod
662 def F2CPreprocessing(PreprocessingFile):
663 """
664 Replace incompatible Fortran PreProcessing directives with its C++ counterparts.
665
666 @author: schu_a1
667 """
668 try:
669 # Resolving compatibility issue > Python 3.6
670 from re.RegexFlag import MULTILINE
671 except ImportError:
672 # Python 3.5 & lower
673 from re import MULTILINE
674
675 def NegationDirective(matchobj):
676 """
677 Replace .not. with ! (e.g. .not. defined will be !defined)
678 """
679 newString = ""
680 if matchobj.group(1) == ".NOT. ":
681 newString += "!"
682
683 newString += "defined(%s)" % matchobj.group(2)
684 return newString
685
686 def LowerDirective(matchobj):
687 """
688 All initial directives are given in lower cases.
689 """
690 return "#" + matchobj.group()[1:].lower()
691
692 # Definition of pattern to be substituted.
693 decorator = re.compile("^!DEC\$ (?=IF|ELSE|ENDIF|END IF)",MULTILINE)
694 addition = re.compile("^#(.*?)\.AND\.(.*?)", MULTILINE)
695 either = re.compile("^#(.*?)\.OR\.(.*?)", MULTILINE)
696 negation = re.compile("(\.NOT\. )?DEFINED\‍(([A-Za-z0-9_]+)\‍)", MULTILINE)
697 conditionals = re.compile("^#(IF|ENDIF|END IF|ELSE)",MULTILINE)
698 space = re.compile("^#(end if)",MULTILINE)
699
700 ## Load everything into memory
701 # Do not specify encoding here.
702 try:
703 # Default's to system specifics
704 with io.open(PreprocessingFile,"r") as f: stream = f.read()
705 except:
706 # Happens in some windows systems.
707 if Utility.GetPlatform() in ["windows"]:
708 try:
709 ## Try windows specific file format first.
710 with io.open(PreprocessingFile,"r", encoding='cp1252') as f: stream = f.read()
711 except:
712 # If not successful, try to use UTF-8 directly
713 with io.open(PreprocessingFile,"r",encoding="utf-8") as f: stream = f.read()
714 else:
715 # Define encoding explicitly
716 with io.open(PreprocessingFile,"r",encoding="utf-8") as f: stream = f.read()
717
718 # Replace all incompatible patters
719 stream = decorator.sub("#",stream)
720 stream = addition.sub("#\g<1>&&\g<2>", stream)
721 stream = either.sub("#\g<1>||\g<2>", stream)
722 stream = negation.sub(NegationDirective, stream)
723 stream = conditionals.sub(LowerDirective, stream)
724 stream = space.sub("#endif",stream)
725
726 # Replace the input file with the updated version. Specify the encoding.
727 try:
728 # Modern convention
729 with io.open(PreprocessingFile,"w",encoding="utf-8") as f: f.write(stream)
730 except:
731 # Legacy version
732 with open(PreprocessingFile,"w") as f: f.write(stream)
733
734 def AddIncludePath(self, includes):
735 """
736 Define additional include directories containing modules or source files as comma separated list.
737 """
738 self.incdirs.append(includes)
739 self.incdirs = list(Utility.ArbitraryFlattening(self.incdirs))
740 pass
741
742 def AddDependencyPath(self, dependencies):
743 """
744 Define additional directories containing 3rd party libraries as comma separated list.
745 """
746 self.libdirs.append(dependencies)
747 self.libdirs = list(Utility.ArbitraryFlattening(self.libdirs))
748 pass
749
750 def UseLibraries(self, libs):
751 """
752 Define which non-default libraries should be used during linking.
753 """
754 self.libs.append(libs)
755 self.libs = list(Utility.ArbitraryFlattening(self.libs))
756 pass
757
758 def SourcePath(self, path):
759 """
760 Define a new source directory. Input is read from workspace by default.
761 """
762 self.srcdir = path
763 ## Source directory can be parsed as relative or
764 # absolute path w.r.t. to is initial calling script.
766 self.srcdir = os.path.abspath(os.getcwd())
767 pass
768
769 def OutputPath(self, path, files=""):
770 """
771 Define a new output directory. Output is written to the workspace by default.
772 """
773 self.outdir = path
774 ## Output directory can be parsed as relative or
775 # absolute path w.r.t. to is initial calling script.
777 self.outdir = os.path.abspath(os.getcwd())
778 ## List of files copied to the output directory.
779 self.copyfiles.append(files)
780 self.copyfiles = list(Utility.ArbitraryFlattening(self.copyfiles))
781 pass
782
783 def Environment(self, path, script="ifortvars.bat"): # pragma: no cover
784 """
785 Load an additional environment file prior to execution of all commands.
786 """
787 ## Execute an additional bash script prior to all build commands.
788 os.environ['pyx_environment'] = os.path.join(path,script)
789 if path.rsplit("\\",1)[1] == "bin":
790 self.intelpath = path[::-1].replace("bin"[::-1],"",1)[::-1]
791 else:
792 self.intelpath = path
793
794 # Update static MKL library include.
795 for inc in self.incdirs:
796 if all(x in inc for x in ["mkl","include"]):
797 self.incdirs.remove(inc)
798
799 # Update static MKL library.
800 for lib in self.libdirs:
801 if all(x in lib for x in ["mkl", "lib"]) or all(x in lib for x in ["compiler","lib"]):
802 self.libdirs.remove(lib)
803
804 # Redefine MKL paths
805 mkl_include_path = os.path.join(self.intelpath,"mkl","include")
806 mkl_lib_path = os.path.join(self.intelpath,"mkl","lib",os.environ['pyx_intel'])
807 mkl_compiler_path = os.path.join(self.intelpath,"compiler","lib",os.environ['pyx_intel'])
808 # Add newly created path to library path.
809 self.incdirs.append(mkl_include_path)
810 self.libdirs.extend([mkl_lib_path, mkl_compiler_path])
811 pass
812
813 def Preprocessing(self, cmdstring='', inend='', outend='', copyfiles=[],
814 replace = {'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'}):
815 """
816 Assemble command string for the pre-build event.
817 """
818 # Space delimiter
819 delimn = " "
820
821 # Add file extension to output file name (if not already been done)
822 if not Utility.IsNotEmpty(os.path.splitext(self.buildname)[1]):
823 self.buildname = self.buildname+outend
824
825 # Add source directory to list of include dirs in case of any preprocessing event.
826 if self.srcdir not in self.incdirs: self.incdirs += [self.srcdir]
827
828 # Go into scratch directory (if defined)
830 # Create two temporary file names.
831 collsrcfile = Utility.GetTemporaryFileName(filename="coll" + "_", extension=inend)
832 presrcfile = Utility.GetTemporaryFileName(filename="pre",extension=outend)
833
834 # Create a list of all directories to be searched
835 search_dir = [self.srcdir]
836 try:
837 search_dir.extend(self.incdirs)
838 except:
839 pass
840
841 # Copy all relevant files from the source folder into the current scratch folder.
842 if copyfiles != []:
843 for subdir in search_dir:
844 # Ignore non-existing folders
845 if not os.path.exists(subdir): continue
846 src_files = os.listdir(subdir)
847 file_names = [x for x in src_files if x in copyfiles]
848 for inputfile in file_names:
849 full_file_name = os.path.join(subdir, inputfile)
850 if (os.path.isfile(full_file_name)):
851 shutil.copy(full_file_name, os.getcwd())
852 self.temps = self.temps + (inputfile, )
853 # Rename source code file
854 else:
855 Utility.ConcatenateFiles(self.buildname,self.srcs,self.srcdir, inend)
856
857 # Do not apply any pre-processing (required for e.g. Beos)
858 if cmdstring != '':
859 # Concatenate all source files into one temporary file
860 Utility.ConcatenateFiles(collsrcfile, self.srcs, self.srcdir, ending=inend)
861
862 # Replace pre-processing commands
863 Utility.ReplaceTextinFile(collsrcfile, presrcfile, replace, source=self.scrtdir)
864
865 # Always align commands with C++.
866 Make.F2CPreprocessing(presrcfile)
867
868 # Add all include directories to the call
869 if any(x in cmdstring for x in ["cpp", "fpp"]):
870 cmdstring += delimn + delimn.join(['-I"'+x+'" ' for x in self.incdirs])
871
872 # Assemble command string
873 cmdstring += ' %s %s' % (presrcfile, self.buildname)
874
875 # Store command string in object
876 ## Command executed during pre-build event.
877 self.precmd = self.iniCompiler+" "+cmdstring
878
879 # Add temporary files to tuple scheduled for removal
880 self.temps = self.temps + (collsrcfile, presrcfile, self.buildname, )
881 pass
882
883 def Build(self, cmdstring, **kwargs): # pragma: no cover
884 """
885 Assemble command string for the main build event.
886 """
887 # Initialize command string
888 cmd = "";
889
890 # Only apply linking and include directories for supported make operations
891 if not self.MakeObjectKind.lower() in ["doxygen"]:
892
893 # Add all include paths to the command string
894 includes = ['-I"'+x+'" ' for x in self.incdirs]
895 for x in includes:
896 cmd += x
897
898 # Add all dependency paths to the command string
899 dependencies = ['-L"'+x+'" ' for x in self.libdirs]
900 for x in dependencies:
901 cmd += x
902
903 # Add all required libraries to the command string
904 libs = ['-l'+x+' ' for x in self.libs]
905 for x in libs:
906 cmd += x
907
908 ## Command line arguments passed in by the user.
909 self.compargs = cmdstring
910 ## Command executed during build event.
911 self.makecmd = self.iniCompiler+" "+os.path.join(self.path2exe,self.exe)+" "+ cmd + cmdstring
912 pass
913
914 def Postprocessing(self, cmdstring=''): # pragma: no cover
915 """
916 Assemble command string for the post-build event.
917 """
918 ## Command executed during post-build event.
919 self.postcmd = self.iniCompiler+" "+cmdstring
920 pass
921
922 @staticmethod
923 def sanitize(string, **kwargs): # pragma: no cover
924 """
925 Provide a dictionary with substrings to replace in the given input.
926
927 @note: Defaults to replace architecture and platform identifiers
928 """
929 result = copy.deepcopy(string)
930 replacements = kwargs.get("replace",{'{arch}': Utility.GetArchitecture(), "{platform}":Utility.GetPlatform()})
931 for key, value in replacements.items():
932 result = result.replace(key, value)
933 result = Utility.GetSanitizedDataFromCommand([result], is_path=False)[-1]
934 return result
935
936 @staticmethod
937 def setup(*args, **kwargs): # pragma: no cover
938 """
939 Initialize a predefined compiler tool chain of all requirements are met.
940
941 @note: Only meaningful on NT systems.
942 """
943 # Future proof by only importing
944 try: from packaging.version import Version as StrictVersion
945 except ImportError: from distutils.version import StrictVersion
946
947 delimn = " "
948
949 # The function is only meaningful on NT systems
950 if Utility.GetPlatform() not in ["windows"]: return
951
952 # Initialize and create tool chain for MSYS2 on Windows
953 if kwargs.get("mingw",False):
954 # Fetch path to choco directory and determine installation directory of msys64
955 _, choco_base_path = Utility.GetExecutable("choco", get_path=True)
956 # There are two executables of chocolatey present within one installation folder. Because reasons.
957 if "bin" not in os.listdir(os.path.dirname(choco_base_path)): choco_base_path = os.path.join(choco_base_path,'..', '..')
958 else: choco_base_path = os.path.join(choco_base_path,'..')
959 choco_base_path = os.path.normpath(choco_base_path)
960 content = open(os.path.join(choco_base_path,"logs","choco.summary.log"),"r").readlines()
961 # Determine msys64 installation directory
962 try: _, msys2_base_path = next(iter([x for x in content if all([y in x.lower() for y in ["msys64","software installed to"]])])).split("Software installed to")
963 # Well, they changed the logs...
964 except StopIteration: _, msys2_base_path = next(iter([x for x in content if all([y in x.lower() for y in ["msys64","installing to"]])])).split("Installing to:")
965 # Get base path
966 try: msys2_base_path = Utility.GetSanitizedDataFromCommand("r"+msys2_base_path.strip(), allowed_exceptions=(ValueError,))[0]
967 except SyntaxError: msys2_base_path = msys2_base_path.strip()
968 msys2_base_path = os.path.normpath(msys2_base_path)
969 # Get compiler path. Default to the latest version
970 msys2_compiler_version = sorted(os.listdir(os.path.join(msys2_base_path,"mingw64","lib","gcc","x86_64-w64-mingw32")),key=StrictVersion)[-1]
971 msys2_compiler_path = os.path.join(msys2_base_path,"mingw64","lib","gcc","x86_64-w64-mingw32",msys2_compiler_version)
972 # Get base initialization shell script
973 msys2_shell_initialization = delimn.join([os.path.join(msys2_base_path,"msys2_shell.cmd"),"-defterm","-mingw64","-no-start","-full-path","-here"])
974 # Return all paths
975 return (msys2_base_path, msys2_compiler_path, msys2_shell_initialization)
976 # Initialize and create tool chain for Visual Studio on Windows
977 elif kwargs.get("msvsc",'vs2015') in ["vs2015","vs2017","vs2019","vs2022"]: # pragma: no cover
978 # Set Intel Fortran Compler path to None as default value
979 intel_compiler_path = ""
980 # Get base initialization batch script
981 msvsc_shell_initialization = r'"'+os.path.join(PyXMakePath,"Build","cmd","windows","iniCompiler.bat")+'"'
982 # Remove support for Paths.log
983 if not any([Utility.GetExecutable(x) for x in ["ifx","ifort"]]) and not os.path.exists(os.path.join(Utility.GetPyXMakePath(),"Paths.log")):
984 # Get directory of the windows start menu
985 allprograms = os.path.join(os.environ['ALLUSERSPROFILE'],"Microsoft","Windows","Start Menu", "Programs")
986 # Search for Intel
987 for root, _, files in Utility.PathWalk(allprograms):
988 if "intel" not in root.lower(): continue
989 if not any(x.lower().endswith("lnk") for x in files) or not any("compiler" in x.lower() for x in files): continue
990 intellink = os.path.abspath(os.path.join(root,sorted(files)[-1]))
991 # Read command from binary link
992 try: command = Utility.GetLink(intellink)
993 # We found nothing associated with Intel. Skip the rest an issue an error later, if required.
994 except UnboundLocalError: return (None, intel_compiler_path, msvsc_shell_initialization )
995 intelpath = next(iter(x for x in command.split('"') if "compiler" in x.lower()))
996 # Set environment variables
997 if not os.getenv("pyx_intel","") and not os.getenv("pyx_msvsc",""):
998 os.environ["pyx_intel"], os.environ["pyx_msvsc"] = [x.replace('"',"") for x in command.split(delimn)[-2:]]
999 # Set environment explicitly
1000 os.environ["pyx_environment"] = intelpath
1001 # Set default Intel Fortran Compiler path
1002 intel_compiler_path = os.path.abspath(os.path.join(intelpath, os.path.pardir, os.path.pardir))
1003 # Attempt to initialize Intel Fortran compiler and read environment data from the call
1004 data = Utility.GetEnvironmentFromCommand(os.path.join(Utility.GetPyXMakePath(),"Build","cmd","windows","iniCompiler.bat"))
1005 # Update the current environment
1006 os.environ.update(data)
1007 ## Compiler executables are already in the path variable of the current process.
1008 # Do not attempt to set the environment here
1009 if any([Utility.GetExecutable(x) for x in ["ifx","ifort"]]): os.environ["pyx_environment"] = "unset"
1010 # Return all paths
1011 return (None, intel_compiler_path, msvsc_shell_initialization )
1012 pass
1013
1014 @classmethod
1015 def run(cls, **kwargs):
1016 """
1017 Assemble command string for the post-build event.
1018 """
1019 ## Run a given class with its default CLI settings when supported.
1020 if hasattr(cls, "parse"): cls.parse(command=sys.argv, **kwargs)
1021 else: RuntimeWarning("Class %s has no associated default parsing feature. Skipping." % str(cls.__name____name__)) # pragma: no cover
1022 pass
1023
1024 def create(self, **kwargs):
1025 """
1026 Execute make command
1027 """
1028 # Dictionary holding all local output settings
1029 settings = {"verbosity": self.verbose, "collect": not "cmake" in self.exe or Utility.GetPlatform() in ["linux"]}
1030
1031 # Execute again to account for input added after compiling command
1032 try: self.Build(self.compargs, **kwargs)
1033 except: pass
1034
1035 # Go into scratch directory (if defined)
1037 ## All files are automatically added to the tuple of temporary files. Thus, these files is this list
1038 # were already present when the process has started. Accordingly, do not delete these files since
1039 # one cannot be sure the scratch directory was set to a workspace with import files
1040 vault = [x for x in os.listdir(os.getcwd()) if x not in self.temps]
1041
1042 for key in os.environ.keys():
1043 # Issue a warning if one of the environment variables pass their maximum limit on windows. Subsequent errors may occur.
1044 if Utility.GetPlatform() == "windows" and self.verbose >= 3 and len(os.environ[key]) >= 2048: # pragma: no cover
1045 warnings.warn("Environment variable %s" % key + " is exceeding the character limit") #@UndefinedVariable
1046
1047 # Pre-build event (if required)
1048 try:
1049 if self.precmd != '':
1050 Utility.Popen(self.precmd, **settings)
1051 except:
1052 pass
1053
1054 # Build event (if required)
1055 try: # pragma: no cover
1056 if getattr(self,"mkl_dependency",""):
1057 # Add additional MKL dependencies
1058 for x in getattr(self,"_mkl_includes",[]):
1059 if x not in np.atleast_1d(self.mkl_dependency): continue
1060 with open("pyx_bridge_"+str(x), "a+") as mkl_include: mkl_include.write(" include '%s' \n" % str(x))
1061 command = self.iniCompiler + " "+'ifort -c "%s"' % str("pyx_bridge_"+str(x))
1062 Utility.Popen(command, **settings)
1063
1064 if self.makecmd != '':
1065 # Modify build runtime environment
1066 env = os.environ.copy();
1067 # PYTHONPATH may not exist
1068 if not "python" in self.makecmd: env.pop("PYTHONPATH","") #@UndefinedVariable
1069 Utility.Popen(self.makecmd, env=env, **settings)
1070 except: pass
1071
1072 # Post-build event (if required)
1073 try:
1074 if self.postcmd != '':
1075 Utility.Popen(self.postcmd, **settings)
1076 except:
1077 pass
1078
1079 # Copy files to predefined output location. Add the original to list of redundant files.
1080 # Copy only those files which are not temporary and whose name includes the BuildID.
1081 # If a list of files if given, use only those files presented in the list. Add to other files to list
1082 # of redundant files.
1083 try: # pragma: no cover
1084 if self.outdir != os.getcwd():
1085 for f in os.listdir(os.getcwd()):
1086 # Do not process any file in vault
1087 if f in vault: continue
1088 # All files from here are created during the process
1089 elif self.architecture in f and f != self.buildname and f not in self.temps:
1090 if self.copyfiles[0] == "":
1091 copyfile(os.path.join(os.getcwd(),f), os.path.join(self.outdir,f))
1092 self.temps = self.temps + (f, )
1093 if f in self.copyfiles:
1094 copyfile(os.path.join(os.getcwd(),f), os.path.join(self.outdir,f))
1095 self.temps = self.temps + (f, )
1096 elif f not in self.copyfiles and self.copyfiles[0] != "" and not os.path.isdir(f):
1097 self.temps = self.temps + (f, )
1098 except: pass
1099 # Finish and delete redundant files
1100 Utility.DeleteFilesbyEnding(self.temps)
1101 pass
1102
1103## @class PyXMake.Build.Make.Custom
1104# Base class for all custom build events inherited from Make.
1105class Custom(Make):
1106 """
1107 Inherited class to build projects without any presets.
1108 """
1109 def __init__(self, *args, **kwargs):
1110 """
1111 Initialization of Custom class object.
1112 """
1113 super(Custom, self).__init__(*args, **kwargs)
1114 ## String identifier of current instance.
1115 self.MakeObjectKind = "Custom"
1116
1117 ## The executable command used in all build events.
1118 self.exe = "cmd.exe /c &&"
1119
1120 ## Change default executable settings when source is a CMAKE file
1121 if self.srcs[-1] in ["CMakeLists.txt"]:
1122 settings = { "search_paths": os.pathsep.join([os.getenv("PATH",os.getcwd()),os.path.join(sys.prefix,"scripts")]) }
1123 # Ensure that the latest GNU compiler is fetched by CMAKE.
1124 self.exe = ["cmake" if not Utility.GetExecutable("cmake", **settings) else
1125 Utility.InQuotes(Utility.GetExecutable("cmake",get_path=True, **settings)[-1]) ][0]
1126 # Convert an absolute CMAKE path to POSIX style format to properly work with MingW64
1127 if Utility.GetPlatform() in ["windows"] and Utility.GetExecutable("choco") and self.hasFosshasFosshasFoss: # pragma: no cover
1128 self.exe = Utility.GetPathConversion(self.exe, "linux")
1129
1130 ## Immutable settings for Custom object.
1131 # Temporary build name, assembled using BuildID.
1133
1134 # Set environment variables for ABAQUS builds (Defaults to latest version).
1135 os.environ["ABQ_ENV_FILE"] = "abaqus_v6.env"
1136 os.environ["pyx_abaqus"] = os.getenv("pyx_abaqus","abaqus")
1137
1138 # Add additional MKL dependencies. Only when MKL support is enabled.
1139 self.mkl_dependencymkl_dependency = ["mkl_vsl.f90"]
1140
1141 ## Command line arguments passed in by the user.
1143
1144 # Add temporary files to tuple scheduled for removal.
1145 self.tempstemps = self.tempstemps + (os.getenv("ABQ_ENV_FILE"), )
1146 pass
1147
1148 def Build(self, cmdstring, **kwargs):
1149 """
1150 Assemble command string for the main build event.
1151 """
1152 delimn = " "
1153
1154 # Do this part only once!
1155 if self.compargscompargs == cmdstring:
1156 # Add all include paths to the include environment variable
1157 os.environ["INCLUDE"] = os.pathsep.join(list(self.incdirs)) + os.pathsep + os.pathsep.join((os.environ.get("MSMPI_INC",""), os.environ.get("INCLUDE","")))
1158
1159 # Add all additional library paths to the lib environment variable
1160 os.environ["LIB"] = os.pathsep.join(self.libdirs) + os.pathsep + os.pathsep.join((os.environ.get("MSMPI_LIB64",""), os.environ.get("LIB","")))
1161
1162 # Add scratch and sources to path environment variable
1163 os.environ["PATH"] = os.pathsep.join([self.srcdir,self.scrtdir]) + os.pathsep + os.getenv("PATH","")
1164
1165 ## Execute a CMake build script
1166 if "cmake" in self.exe:
1167
1168 # Create build command
1169 command = "%s" % self.exe
1170 # Source path quotation is dependent of the underlying system build framework
1171 if not self.hasFosshasFosshasFoss and Utility.GetPlatform() in ["windows"]: command += ' -B build -S '"%s"'' % os.path.abspath(self.srcdir)
1172 else: command += " -B build -S '%s'" % os.path.abspath(self.srcdir)
1173 # Remove duplicate cmake call
1174 command += cmdstring.replace("cmake"," ")
1175
1176 ## Attribute only contains string when a non-default output directory has been defined.
1177 # Update the installation prefix in that case
1178 if any(isinstance(x,six.string_types) for x in getattr(self,"copyfiles",[])): # pragma: no cover
1179 command += " -DCMAKE_INSTALL_PREFIX='%s'" % os.path.abspath(self.outdir)
1180
1181 # Collect half the number of available core to speed up the build process
1182 cpus = str(multiprocessing.cpu_count()//2) #@UndefinedVariable
1183
1184 # Deactivate MKL dependency
1186
1187 # Explicitly set all compilers
1188 if self.hasFosshasFosshasFoss: command +=" -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_Fortran_COMPILER=gfortran"
1189
1190 # Add verbose output
1191 if self.verbose >= 2: command +=" -DCMAKE_VERBOSE_MAKEFILE=On"
1192
1193 # Add all additional supplied user options w/o further verification
1194 if kwargs.get("append",[]): command += delimn + delimn.join(kwargs.get("append"))
1195
1196 # Run CMake on windows with cross compilation
1197 if not self.hasFosshasFosshasFoss and Utility.GetPlatform() in ["windows"]: # pragma: no cover
1198 command +=' -DCMAKE_BUILD_TYPE=Release'
1199 # Custom header files are required on windows.
1200 command +=' -DCMAKE_INCLUDE_PATH="%s"' % os.path.join(PyXMakePath,"VTL","make")
1201 # Add local script directory to local search path
1202 settings = {"search_paths":os.pathsep.join([os.path.join(sys.prefix,"Scripts"),os.getenv("PATH","")])}
1203 # Help CMAKE to find a compatible SED program
1204 if Utility.GetExecutable("pythonsed",**settings):
1205 command +=' -DSED="%s"' % Utility.GetExecutable("pythonsed", get_path=True, **settings)[-1]
1206 elif Utility.GetExecutable("choco"):
1207 try:
1208 settings = {"search_paths":os.pathsep.join([os.path.join(Make.setup(mingw = Utility.GetExecutable("choco"))[0],"usr","bin"),os.getenv("PATH","")])}
1209 command +=' -DSED="%s"' % Utility.GetExecutable("sed", get_path=True, **settings)[-1]
1210 except: pass
1211 # If SED is found directly. Do nothing
1212 elif Utility.GetExecutable("sed"): pass
1213 # Support pythonSED executable
1214 # Should never happen
1215 else: pass
1216 # Suppress warnings for developers and set up NMAKE
1217 command +=' -Wno-dev -G "NMake Makefiles"'
1218 # When using Intel oneAPI environment, use classic compiler for the time being
1219 if Utility.GetExecutable("oneapi-cli") and Utility.GetExecutable("ifort"): command += " -DCMAKE_Fortran_COMPILER=ifort"
1220 # Get batch script initialization
1221 batch = delimn.join([self.iniCompiler,"&&"])
1222 # Define build commands
1223 self.precmdprecmd = delimn.join([batch,'%s' % command])
1224 self.makecmdmakecmd = delimn.join([batch,'%s' % "%s --build build" % self.exe])
1225 self.postcmdpostcmd = delimn.join([batch,'%s' % "%s --install build" % self.exe])
1226 # Run using MINGW on NT systems
1227 elif Utility.GetPlatform() in ["windows"] and Utility.GetExecutable("choco"): # pragma: no cover
1228 command += " -G 'MinGW Makefiles'"
1229 # Get MSYS2 initialization
1230 msys2_shell = delimn.join([self.setup(mingw=True)[-1],"-c"])
1231 # Define build commands
1232 self.precmdprecmd = delimn.join([msys2_shell,'"%s"' % command])
1233 self.makecmdmakecmd = delimn.join([msys2_shell,'"%s"' % "%s --build build -j %s" % (self.exe, cpus)])
1234 self.postcmdpostcmd = delimn.join([msys2_shell,'"%s"' % "%s --install build" % self.exe])
1235 # Run CMake on POSIX systems
1236 else:
1237 # Define build commands
1238 self.precmdprecmd = shlex.split(command, posix=not os.name.lower() in ["nt"])
1239 self.makecmdmakecmd = shlex.split("%s --build build -j %s" % (self.exe, cpus), posix=not os.name.lower() in ["nt"])
1240 self.postcmdpostcmd = shlex.split("%s --install build" % self.exe, posix=not os.name.lower() in ["nt"])
1241
1242 # Add pre-processed source file to environment variable
1243 os.environ["pyx_file"] = self.buildnamebuildname
1244
1245 # Add exception to pyx_libs when ABAQUS build command is used with vs2015 and higher.
1246 if self.msvsc.lower() in ["vs2015", "vs2017","vs2019","vs2022"]:
1247 self.libs.extend(["msvcrt", "vcruntime", "ucrt", "legacy_stdio_definitions"])
1248 # Remove explicit reference to Microsoft Runtime Library when using lastest Intel OneAPI environment
1249 if Utility.GetExecutable("oneapi-cli"): self.libs.remove("msvcrt")
1250
1251 # Add all libraries to a environment variable
1252 pyx_libs = ["'"+x+".lib'" for x in sorted(set(self.libs), key=self.libs.index)]
1253
1254 # Set environment variable
1255 os.environ["pyx_libs"] = ",".join(pyx_libs)
1256
1257 ## Command line arguments passed in by the user.
1258 self.compargscompargs = cmdstring
1259 ## Command executed during build event.
1260 if not getattr(self, "makecmd",""): self.makecmdmakecmd = self.iniCompiler+" "+self.exe+" "+cmdstring
1261
1262 # Add temporary files to tuple scheduled for removal
1263 self.tempstemps = self.tempstemps + (self.buildnamebuildname, )
1264 pass
1265
1266 @classmethod
1267 def parse(cls, **kwargs): # pragma: no cover
1268 """
1269 Execute the current class as a CLI command.
1270 """
1271 # Import its main from VTL
1272 from PyXMake.VTL import cmake
1273 # Evaluate current command line
1274 command = kwargs.get("command",sys.argv)
1275 # Process all known arguments
1276 parser = argparse.ArgumentParser(description='CLI wrapper options for CMAKE with more sensible default settings.')
1277 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
1278 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
1279 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for CMake. Defaults to current workspace.")
1280 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
1281 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
1282 parser.add_argument("--foss", type=Utility.GetBoolean, const=True, default=True, nargs='?',
1283 help="Toggle to enforce free-open source builds. Defaults to True.")
1284 # Check all options or run unit tests in default mode
1285 try:
1286 # Check CLI options
1287 _ = command[1]
1288 args, unknown = parser.parse_known_args(command[1:])
1289 # Project name is mandatory
1290 project = args.name[0];
1291 # Specification of source directory is mandatory
1292 source = args.source[0] ;
1293 # Optional non-default output directory
1294 try: output = args.output[0]
1295 except: output = None
1296 # Optional non-default scratch directory
1297 try: scratch = args.scratch[0]
1298 except: scratch = os.path.abspath(os.getcwd())
1299 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
1300 try: verbosity = int(args.verbosity[0])
1301 except: verbosity = 0
1302 # Optional non-default package check
1303 try: foss = args.foss[0]
1304 except: foss = Utility.GetPlatform() in ["linux"]
1305 # Create a dictionary combining all settings
1306 settings = {"source":source, "output":output, "scratch":scratch,
1307 "verbosity":verbosity, "foss": foss}
1308 # Parse all unknown command line parameter directly to cmake
1309 append = [x for x in unknown if not any(y in x for y in vars(args).keys())]
1310 if append: settings.update({"append":append})
1311 # Use an exception to allow help message to be printed.
1312 except Exception as _:
1313 # Local imports. These are only meaningful while executing an unit test
1314 from PyCODAC.Tools.Utility import GetPyCODACPath
1315 # Build all supported features
1316 if AllowDefaultMakeOption:
1317 # Run compilation of MCODAC using CMake
1318 BuildID = "mcd_core";
1319 # Compile everything using CMake.
1320 cmake(BuildID, source=os.path.join(GetPyCODACPath(),"Core","config"), foss=kwargs.pop("foss",True))
1321 else:
1322 # Execute CLI command
1323 cmake(project, **settings)
1324 pass
1325
1326## @class PyXMake.Build.Make.CCxx
1327# Base class for all C/C++ build events inherited from Make.
1328class CCxx(Make,NT,POSIX):
1329 """
1330 Inherited class to build projects using Intel C/C++.
1331 """
1332 def __init__(self, *args, **kwargs):
1333 """
1334 Initialization of C/C++ class object.
1335 """
1336 super(CCxx, self).__init__(*args, **kwargs)
1337 ## String identifier of current instance.
1338 self.MakeObjectKind = 'CCxx'
1339
1340 ## The executable command used in the main build event.
1341 self.exe = 'cl.exe'; os.environ["pyx_compiler"] = "gcc"
1342
1343 ## Static or dynamic link library flag.
1344 self.isStaticisStatic = True if kwargs.get('lib', 'static') not in ['shared', 'SHARED', 'Shared'] else False
1345
1346 ## Define if the input should be compiled exactly as provided.
1347 # Defaults to False, meaning that merging & pre-processing utilities will be carried out.
1348 self.incrementalincremental = kwargs.get('incremental', False)
1349
1350 # Immutable settings for C/Cpp object
1351 if self.incrementalincremental:
1352 self.exe += ' -c %s' % (' '.join(self.srcs))
1353 else: # pragma: no cover
1354 self.exe += ' -c %s' % (self.buildnamebuildname)
1355
1356 ## Name of library, assembled using BuildID.
1357 self.libname = self.buildid + self.architecture
1358 ## Temporary build name.
1360
1361 # Initialization of lists containing additional sources, modules or libraries
1362 ## List of libraries which should be statically linked in.
1363 self.linkedIn = []
1364
1365 # Initialization of tuple containing temporary files
1366 ## Blank version of tuple to store temporary file names scheduled for removal.
1368
1369 # Remove MKL from default command line
1370 ## Blank version of list containing library directories without initially specifying MKL.
1372
1373 # Always add conversion headers to the default make directory
1374 self.incdirs.append(os.path.join(PyXMakePath,"VTL","make"))
1375
1376 # Identify source code and BuildID and set the corresponding environment variables for Mingw64 and Linux.
1377 if kwargs.get("bash", self.hasFosshasFoss):
1378 os.environ["pyx_buildid"], os.environ["pyx_source"] = (self.libname, ' '.join([x for x in self.srcs if os.path.splitext(x)[1].lower() in (".c", ".cpp", ".h", ".hpp", "Makefile")]))
1379 pass
1380
1381 def OutputPath(self, libpath=os.getcwd()):
1382 """
1383 Define output directories for modules and libraries.
1384 """
1385 ## Output path for library files.
1386 self.outlibsoutlibs = libpath
1387 pass
1388
1389 def Build(self, cmdstring, **kwargs):
1390 """
1391 Assemble command strings for the main build event.
1392 """
1393 # Initialize command string
1394 cmd = ""
1395
1396 # Add all include paths to the command string
1397 includes = [' -I"'+x+'" ' for x in self.incdirs]
1398 for x in includes:
1399 cmd += x
1400
1401 # Choose the librarian and the file extension of the library.
1402 if not self.isStaticisStatic: # pragma: no cover
1403 librarian = 'link -dll -fixed:no -defaultlib:libcmt.lib -nodefaultlib:msvcrt.lib '
1404 ext = '.dll'
1405 else:
1406 librarian = 'lib '
1407 ext = '.lib'
1408
1409 # Build commands using Intel Fortran (immutable)
1410 ## Used defined command line options.
1411 self.compargscompargs = cmdstring
1412 ## Intel Compiler command.
1414 self.makecmdmakecmd += self.exe + cmd + cmdstring + ' && '
1415 ## Intel Linker command.
1416 self.linkcmd = librarian +'*.obj -nologo -machine:'+self.architecture+' -out:'+os.path.join(self.outlibsoutlibs,self.libname+ext)
1417
1418 # Add temporary files to tuple scheduled for removal
1419 self.tempstempstemps = self.tempstempstemps + (self.libname+'.obj', ".obj",)
1420 pass
1421
1422 @classmethod
1423 def parse(cls, **kwargs): # pragma: no cover
1424 """
1425 Execute the current class as a CLI command.
1426 """
1427 # Import its main from VTL
1428 from PyXMake.VTL import cxx
1429 # Evaluate current command line
1430 command = kwargs.pop("command",sys.argv)
1431 # Process all known arguments
1432 parser = argparse.ArgumentParser(description="Build a sCxx project from console.")
1433 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
1434 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
1435 parser.add_argument('-f', '--files', nargs='+', default=[], help="Source file or list of all source files in the order of compilation")
1436 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
1437 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional directories and files required for the build.")
1438 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for the build. Defaults to current workspace.")
1439 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
1440 parser.add_argument("--incremental", type=Utility.GetBoolean, const=True, default=False, nargs='?',
1441 help="Toggle between incremental and non-incremental build. Defaults to False.")
1442
1443 try:
1444 # Check CLI options
1445 _ = command[1]
1446 args, _ = parser.parse_known_args(command[1:])
1447 # Project name is mandatory
1448 project = args.name[0]
1449 # Specification of source directory is mandatory
1450 source = args.source[0] ;
1451 # Optional non-default output directory
1452 try: files = args.files
1453 except: files = []
1454 # Optional non-default output directory
1455 try: output = args.output[0]
1456 except: output = os.path.abspath(os.getcwd())
1457 # Optional non-default definition of additional tests cases
1458 try:
1459 _ = args.include[0]
1460 # Collect all given paths. Get system independent format
1461 include = Utility.GetSanitizedDataFromCommand(args.include)
1462 # No extra test cases have been given
1463 except: include = []
1464 # Optional incremental build option. Defaults to False.
1465 try: incremental = args.incremental[0]
1466 except: incremental = False
1467 # Optional non-default scratch directory
1468 try: scratch = args.scratch[0]
1469 except: scratch = os.path.abspath(os.getcwd())
1470 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
1471 try: verbosity = int(args.verbosity[0])
1472 except: verbosity = 0
1473 # Create a dictionary combining all settings
1474 settings = {"files":files, "incremental": incremental, "include": include,
1475 "source":source, "output":output, "scratch":scratch,
1476 "verbosity":verbosity}
1477 # Use an exception to allow help message to be printed.
1478 except Exception as _:
1479 # Build all supported features for current Python version (default options)
1480 if AllowDefaultMakeOption:
1481 ## This part serves as a unit test. It it not executed by default when installed from PyPi
1482 try:
1483 # Build Muesli with default settings.
1484 BuildID = "muesli"; cxx(BuildID, foss=False, **kwargs)
1485 except: pass
1486 try:
1487 # Build DispLam
1488 from PyCODAC.Plugin.DispLam import __path__ as DispLamPath
1489 disp_src = os.path.join(DispLamPath[0],"src","displam"); disp_bin=os.path.join(DispLamPath[0],"bin",Utility.GetPlatform())
1490 cxx("displam", os.listdir(disp_src), source=disp_src, output=disp_bin, verbosity=0, **kwargs)
1491 except: pass
1492 # Execute valid CLI command
1493 else: cxx(project, **settings)
1494 pass
1495
1496 def create(self): # pragma: no cover
1497 """
1498 Execute make command
1499 """
1500 cmd = ''
1501
1502 # Go into scratch directory (if defined)
1504
1505 # Pre-build event (if required)
1506 try:
1507 if self.precmdprecmdprecmdprecmd != '':
1508 Utility.Popen(self.precmdprecmdprecmdprecmd, self.verboseverboseverbose)
1509 except:
1510 pass
1511
1512 # Add all dependency paths to the command string
1513 dependencies = [' -LIBPATH:"'+x+'" ' for x in self.libdirslibdirslibdirs]
1514 for y in dependencies:
1515 cmd += y
1516
1517 # Add libraries for linking to the command string
1518 linklist = [' '+x+'.lib ' for x in self.libs]
1519 for x in linklist:
1520 cmd += x
1521
1522 # Build event (if required)
1523 try:
1524 if self.makecmdmakecmd != '':
1525 # Execute again to account for input added after compiling command
1526 self.BuildBuild(self.compargscompargs)
1527 Utility.Popen(self.makecmdmakecmd+self.linkcmd+cmd, self.verboseverboseverbose)
1528 except Exception as _: pass
1529
1530 # Post-build event (if required)
1531 try:
1532 if self.postcmdpostcmd != '':
1533 Utility.Popen(self.postcmdpostcmd, self.verboseverboseverbose)
1534 except:
1535 pass
1536
1537 # Finish and delete redundant files
1538 if not self.isStaticisStatic:
1539 os.remove(os.path.join(self.outlibsoutlibs,self.libname+'.exp'))
1540 os.remove(os.path.join(self.outlibsoutlibs,self.libname+'.lib'))
1541 Utility.DeleteFilesbyEnding(self.tempstempstemps)
1542 pass
1543
1544## @class PyXMake.Build.Make.Fortran
1545# Base class for all Fortran build events.
1546# Inherited from Make and flavors in dependence of the underlying or requested operating system (optionally).
1547class Fortran(Make,NT,POSIX):
1548 """
1549 Inherited class to build projects using Intel Fortran.
1550 """
1551 def __init__(self, *args, **kwargs):
1552 """
1553 Initialization of Fortran class object.
1554 """
1555 super(Fortran, self).__init__(*args, **kwargs)
1556 ## String identifier of current instance.
1557 self.MakeObjectKind = 'Fortran'; os.environ["pyx_compiler"] = "gfortran"
1558
1559 ## Static or dynamic link library flag.
1560 self.isStaticisStatic = True if kwargs.get('lib', 'static') not in ['shared', 'SHARED', 'Shared'] else False
1561
1562 # Add static MKL libraries when creating a shared resource library (for Java applications).
1563 if not self.isStaticisStatic: # pragma: no cover
1564 if self.architecturearchitecture == "x86":
1565 mkl_interface_lib = "mkl_intel_c"
1566 else:
1567 mkl_interface_lib = "mkl_intel_lp64"
1568
1569 # Always link statically against MKL library
1570 self.libslibs.append([mkl_interface_lib, "mkl_intel_thread", "mkl_core","libiomp5md"])
1571 self.libslibs = list(Utility.ArbitraryFlattening(self.libslibs))
1572 pass
1573
1574 # Immutable settings for Fortran object
1575 ## Name of library, assembled using BuildID.
1576 self.libname = ("lib" if Utility.GetPlatform() in ["linux"] else "") + self.buildid + self.architecturearchitecture
1577 ## Temporary build name.
1579
1580 # Activate / deactivate incremental linking
1581 self.incrementalincremental = kwargs.get("incremental",False)
1582
1583 # Defined here to be checked later.
1584 ## Wrapper interface file for 3rd party FORTRAN code. Automatically creates a module of the underlying source material.
1586 self.wrapper_source = ""
1587 self.wrapper_modulewrapper_module = "pyx_module.f90"
1588
1589 # Initialization of lists containing additional sources, modules or libraries
1590 ## List of libraries which should be statically linked in.
1591 self.linkedIn = []
1592
1593 # Remove MKL from default command line
1594 ## Blank version of list containing library directories without initially specifying MKL.
1596
1597 # Identify source code and BuildID and set the corresponding environment variables for Mingw64 and Linux.
1598 if kwargs.get("bash", self.hasFosshasFoss):
1599 os.environ["pyx_buildid"], os.environ["pyx_source"] = (self.libname, ' '.join([x for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]))
1600 pass
1601
1602 def OutputPath(self, modulepath=None, libpath=os.getcwd()):
1603 """
1604 Define output directories for modules and libraries.
1605 """
1606 # Output module files to scratch directory by default.
1607 if modulepath is None:
1608 modulepath = self.scrtdirscrtdirscrtdir
1609 ## Output path for module or header files.
1610 self.outmoduleoutmodule = modulepath
1611 ## Output path for library files.
1612 self.outlibsoutlibs = libpath
1613 pass
1614
1615 def Preprocessing(self, inend='', outend='', copyfiles=[],
1616 replace = {'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'},
1617 decorator="!DEC$ ATTRIBUTES DLLEXPORT::"):
1618 """
1619 Assemble command string for the pre-build event.
1620 """
1621 # Avoid false positives when created a shared resource library.
1622 delimn = " "; validater ="bind"
1623
1624 # Save command if already defined
1625 _preprocessing = copy.deepcopy(getattr(self,"precmd",None))
1626
1627 # Execute base class method
1628 Make.Preprocessing(self,"custom", inend, outend, copyfiles, replace)
1629
1630 # Go into scratch directory (if defined)
1632
1633 # Never merge files unless explicitly allowed
1634 if copyfiles:
1635 # File might not exists - use exception to catch that
1636 try: os.remove(os.path.join(os.getcwd(),self.buildnamebuildnamebuildname))
1637 except FileNotFoundError: pass
1638 # Mimic the previous work flow
1639 else:
1640 # Get temporary file name
1641 presrcfile = self.precmdprecmdprecmdprecmd.split(delimn)[-2]
1642
1643 # Check if decorators have to be added to the source file.
1644 Inputfile = os.path.join(os.getcwd(),presrcfile)
1645 Outputfile = os.path.join(os.getcwd(),self.buildnamebuildnamebuildname)
1646 with open(Inputfile) as infile, open(Outputfile, 'w') as outfile: # pragma: no cover
1647 for line in infile:
1648 outfile.write(line)
1649 if not self.isStaticisStatic:
1650 xname, xline = line.partition('="')[0], line.partition('="')[2]
1651 if xline != '' and validater in xname.lower():
1652 outfile.write(decorator+xline.partition('"')[0]+"\n")
1653
1654 # Recover original command
1655 self.precmdprecmdprecmdprecmd = _preprocessing
1656 pass
1657
1658 def Wrapper(self, module_name, source_name=None): # pragma: no cover
1659 """
1660 Assemble command string for the pre-build event.
1661 """
1662 # Add module wrapper to the default make directory
1663 makedir = os.path.join(PyXMakePath,"VTL","make")
1664 TmpFile = Utility.GetTemporaryFileName(extension=str(os.path.splitext(self.wrapper_modulewrapper_module)[1]))
1665 copyfile(os.path.join(makedir,self.wrapper_modulewrapper_module), os.path.join(self.scrtdirscrtdirscrtdir,TmpFile))
1666
1667 if source_name:
1668 self.wrapper_source = source_name
1669
1670 self.intermediate_wrapperintermediate_wrapper = Utility.GetTemporaryFileName(extension=str(os.path.splitext(TmpFile)[1]))
1671
1672 # Go into scratch directory (if defined)
1674 # Prepare wrapper module for later use
1675 Utility.ReplaceTextinFile(TmpFile, self.intermediate_wrapperintermediate_wrapper, {'%pyx_module%':module_name}, source=self.scrtdirscrtdirscrtdir)
1676
1677 # Add temporary files to tuple scheduled for removal
1679 pass
1680
1681 def Build(self, cmdstring, **kwargs): # pragma: no cover
1682 """
1683 Assemble command strings for the main build event.
1684 """
1685 sep = " "; multi_objects = []
1686
1687 # Initialize command string
1688 if not self.incrementalincremental:
1689 cmd = ' -object:'+self.libname+' -module:'+self.outmoduleoutmodule+' -I:"'+self.outmoduleoutmodule+'"'
1690 else:
1691 # Add an trailing separator to indicate a folder
1692 cmd = ' -object:'+self.scrtdirscrtdirscrtdir+os.path.sep+' -module:'+self.outmoduleoutmodule+' -I:"'+self.outmoduleoutmodule+'"'
1693
1694 # Add all include paths to the command string
1695 includes = [' -I"'+x+'" ' for x in self.incdirs]
1696 for x in includes:
1697 cmd += x
1698
1699 # Choose the librarian and the file extension of the library.
1700 if not self.isStaticisStatic:
1701 librarian = 'link -dll -fixed:no -defaultlib:libcmt.lib -nodefaultlib:msvcrt.lib '
1702 ext = '.dll'
1703 else:
1704 librarian = 'lib '
1705 ext = '.lib'
1706
1707 # Build commands using Intel Fortran (immutable)
1708 ## Used defined command line options.
1709 self.compargscompargs = cmdstring
1710 ## Intel Compiler command.
1712
1713 # Check whether an interface module wrapper was added to the current folder
1714 if os.path.isfile(self.intermediate_wrapperintermediate_wrapper):
1715 makefile = "-fpp "+ self.wrapper_modulewrapper_module
1716 if (Utility.IsNotEmpty(self.wrapper_source)):
1718 elif self.incrementalincremental:
1719 makefile = sep.join([x for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")])
1720 multi_objects = [os.path.splitext(x)[0]+".obj" for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]
1721 else:
1722 makefile = self.buildnamebuildnamebuildname
1723
1724 self.makecmdmakecmd += 'ifort -c '+ makefile + cmd + cmdstring + ' && '
1725 ## Intel Linker command.
1726 if not multi_objects:
1727 self.linkcmd = librarian +self.libname+'.obj -nologo -machine:'+self.architecturearchitecture+' -out:'+os.path.join(self.outlibsoutlibs,self.libname+ext)
1728 else:
1729 self.tempstempstemps = self.tempstempstemps + (".obj",)
1730 self.linkcmd = librarian +sep.join(multi_objects) +' -nologo -machine:'+self.architecturearchitecture+' -out:'+os.path.join(self.outlibsoutlibs,self.libname+ext)
1731
1732 # Add temporary files to tuple scheduled for removal
1733 self.tempstempstemps = self.tempstempstemps + (self.libname+'.obj', '.mod')
1734 pass
1735
1736 def create(self, **kwargs): # pragma: no cover
1737 """
1738 Execute make command
1739 """
1740 cmd = ''
1741
1742 # Go into scratch directory (if defined)
1744
1745 # Pre-build event (if required)
1746 try:
1747 if self.precmdprecmdprecmdprecmd != '':
1748 Utility.Popen(self.precmdprecmdprecmdprecmd, self.verboseverboseverbose)
1749 except:
1750 pass
1751
1752 # Add all dependency paths to the command string
1753 dependencies = [' -LIBPATH:"'+x+'" ' for x in self.libdirslibdirslibdirs]
1754 for y in dependencies:
1755 cmd += y
1756
1757 # Add libraries for linking to the command string
1758 linklist = [' '+x+'.lib ' for x in self.libslibs]
1759 for x in linklist:
1760 cmd += x
1761
1762 # Delete old module files
1763 if os.path.exists(self.outmoduleoutmodule):
1764 # Only meaningful when output directory for a module already exists
1765 for f in os.listdir(self.outmoduleoutmodule):
1766 if f.endswith('.mod') and not kwargs.get("combine",False):
1767 os.remove(os.path.join(self.outmoduleoutmodule,f))
1768 else: os.makedirs(self.outmoduleoutmodule)
1769
1770 # Build event (if required)
1771 try:
1772 if self.makecmdmakecmd != '':
1773 # Execute again to account for input added after compiling command
1774 self.BuildBuild(self.compargscompargs)
1775 if os.path.isfile(self.intermediate_wrapperintermediate_wrapper):
1776 Utility.ReplaceTextinFile(self.intermediate_wrapperintermediate_wrapper, self.wrapper_modulewrapper_module, {'%pyx_source%':'"'+self.buildnamebuildnamebuildname+'"'}, source=self.scrtdirscrtdirscrtdir)
1777 Utility.Popen(self.makecmdmakecmd+self.linkcmd+cmd, self.verboseverboseverbose)
1778 except:
1779 pass
1780
1781 # Post-build event (if required)
1782 try:
1783 if self.postcmdpostcmd != '':
1784 Utility.Popen(self.postcmdpostcmd, self.verboseverboseverbose)
1785 except:
1786 pass
1787
1788 # Combine event (needed for TOMS). Combine multiple libraries into ONE.
1789 if self.isStaticisStatic and kwargs.get("combine", False):
1790 sep = ' '; librarian = 'lib '; ext = '.lib'
1791 mergedid = os.path.basename(self.outmoduleoutmodule)
1792 multi_libs = [os.path.join(self.outlibsoutlibs,x) for x in [list(Utility.ArbitraryFlattening(x[2])) for x in Utility.PathWalk(self.outlibsoutlibs)][0] if x.startswith(mergedid)]
1793
1794 try:
1795 # Remove old combined library from the list.
1796 multi_libs.remove(os.path.join(self.outlibsoutlibs,mergedid+self.architecturearchitecture+ext))
1797 except:
1798 pass
1799
1800 self.postcmdpostcmd = self.iniCompiler + sep + librarian +sep.join(multi_libs) +' -nologo -machine:'+self.architecturearchitecture+' -out:'+os.path.join(self.outlibsoutlibs,mergedid+self.architecturearchitecture+ext)
1801 Utility.Popen(self.postcmdpostcmd, self.verboseverboseverbose)
1802 for lib in multi_libs:
1803 os.remove(os.path.join(self.outlibsoutlibs,lib))
1804
1805 # Finish and delete redundant files
1806 if not self.isStaticisStatic:
1807 os.remove(os.path.join(self.outlibsoutlibs,self.libname+'.exp'))
1808 os.remove(os.path.join(self.outlibsoutlibs,self.libname+'.lib'))
1809 Utility.DeleteFilesbyEnding(self.tempstempstemps)
1810 pass
1811
1812## @class PyXMake.Build.Make.PyReq
1813# Base class for all PyReq build events. Inherited from Custom.
1814class PyReq(Custom):
1815 """
1816 Inherited class to build projects using PyReq.
1817 """
1818 def __init__(self, *args, **kwargs):
1819 """
1820 Initialization of PyReq class object.
1821
1822 @note Creates a list of all 3rd party dependencies of a package using PyReq.
1823 """
1824 super(PyReq, self).__init__(*args, **kwargs)
1825 ## String identifier of current instance.
1827
1828 # Remove all default libraries, paths and includes from Make class.
1829 self.libslibs = []
1830 self.libdirslibdirs = []
1831 self.incdirsincdirs = []
1832
1833 # The default is no pre-processing.
1835 # Do not add version by default.
1836 self.compargscompargscompargs = "--no-pin"
1837 pass
1838
1839 def Preprocessing(self, cmdstring=''):
1840 """
1841 Assemble command string for the pre-build event.
1842 """
1843 delimn = " "
1844 # Store command string in object
1845 ## Command executed during pre-build event.
1846 self.precmdprecmdprecmd = delimn.join([self.iniCompiler,cmdstring])
1847 pass
1848
1849 @classmethod
1850 def parse(cls, **kwargs): # pragma: no cover
1851 """
1852 Execute the current class as a CLI command.
1853 """
1854 # Import its main from VTL
1855 from PyXMake.VTL import pyreq
1856 # Evaluate current command line
1857 command = kwargs.get("command",sys.argv)
1858 # Process all known arguments
1859 parser = argparse.ArgumentParser(description='CLI wrapper options for dependency detection tool.')
1860 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
1861 parser.add_argument('--source', type=str, nargs=1, help="Absolute path to project folder.")
1862 parser.add_argument("--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
1863 parser.add_argument("--file", type=str, nargs=1, help="Output file name. Defaults to requirements.txt")
1864 parser.add_argument("--check", type=Utility.GetBoolean, const=True, default=True, nargs='?',
1865 help="Check public PyPi repository to verify the results. Defaults to True.")
1866 parser.add_argument('--command', nargs=argparse.REMAINDER, help="Additional command line parameters.")
1867 # Command line separator
1868 delimn = " "
1869 # Check all options or run unit tests in default mode
1870 try:
1871 # Check CLI options
1872 _ = command[1]
1873 args, _ = parser.parse_known_args(command[1:])
1874 # Project name is mandatory
1875 project = args.name[0]
1876 # Check optional command line arguments
1877 try: source = args.source[0] ;
1878 except:
1879 # Get path from the project name.
1880 handle = importlib.import_module(project)
1881 source = os.path.abspath(handle.__path__[0])
1882 # Optional non-default output directory
1883 try: output = args.output[0]
1884 except: output = os.path.abspath(os.getcwd())
1885 # Optional non-default package check
1886 try: check = args.check[0]
1887 except: check = True
1888 # Optional non-default output filename
1889 try: filename = args.file[0]
1890 except: filename = 'requirements.txt'
1891 # Optional non-default additional command line options for pipreqs.
1892 try: command = args.command[0]
1893 except: command = delimn.join(["--ignore","cmd,bin,Core,Plugin","--no-pin"])
1894 # Use an exception to allow help message to be printed.
1895 except Exception as _:
1896 # Build all supported features
1897 if AllowDefaultMakeOption:
1898 # Command line separator
1899 delimn = " "
1900 # Get absolute path to gather information about package requirements.
1901 from PyXMake import PyXMakePath; from PyCODAC import PyCODACPath #@UnresolvedImport
1902 # Gather latest dependencies from PyXMake's code base.
1903 pyreq("PyXMake", source=PyXMakePath,
1904 compargs=delimn.join(["--ignore","cmd,bin","--no-pin"]), output=os.getcwd(), check=True)
1905 # Gather latest dependencies from PyCODAC's code base.
1906 pyreq("PyCODAC", source=PyCODACPath,
1907 compargs=delimn.join(["--ignore","cmd,bin,Core,Plugin","--no-pin"]), output=os.getcwd(), check=True)
1908 try:
1909 # Gather latest dependencies from DELiS's code base.
1910 pyreq("DELiS", source=os.path.join(PyCODACPath,"Plugin","DELiS","src","delis"),
1911 compargs=delimn.join(["--no-pin"]), output=os.getcwd())
1912 except: pass
1913 try:
1914 # Gather latest dependencies from Smetana's code base.
1915 pyreq("Smetana", source=os.path.join(PyCODACPath,"Plugin","Smetana","src","smetana"),
1916 compargs=delimn.join(["--ignore","gopak,curvefit,test,static","--no-pin"]), output=os.getcwd())
1917 except: pass
1918 # Execute valid CLI command
1919 else: pyreq(project, source, output, check=check, filename=filename, compargs=command)
1920 pass
1921
1922 def create(self,**kwargs):
1923 """
1924 Execute make command
1925 """
1926 # Increase recursion limit (required for OCC)
1927 sys.setrecursionlimit(kwargs.get("recursion_limit",5000))
1928 delimn = " "; linebreak = "\n"
1929
1930 # Go into scratch folder
1932
1933 # Pre-build event (if required)
1934 try: # pragma: no cover
1935 if self.precmdprecmdprecmd.split() > 1 and self.precmdprecmdprecmd != "":
1936 Utility.Popen(self.precmdprecmdprecmd, self.verboseverbose)
1937 except:
1938 pass
1939
1940 # Run build command
1941 data = Utility.GetRequirements(self.srcdir, getattr(self,"compargs","--no-pin").split(delimn), **kwargs)
1942
1943 # Create a summary as a markdown file
1944 try:
1945 with open(os.path.join(self.outdiroutdir,kwargs.get("filename",self.buildid+".md")), "w+") as output:
1946 output.write(linebreak.join(str(item) for item in data))
1947 except:
1948 pass
1949 return
1950
1951## @class PyXMake.Build.Make.Py2X
1952# Base class for all Py2X (for now only f2py) build events.
1953# Inherited from Make and flavors in dependence of the underlying or requested operating system (optionally).
1954class Py2X(Make,NT,POSIX):
1955 """
1956 Inherited class to build projects using Py2X.
1957 """
1958 def __init__(self, *args, **kwargs): # pragma: no cover
1959 """
1960 Initialization of Py2X class object.
1961
1962 @note Currently uses f2py - but should be build with Py2X (DLR) in the future
1963 """
1964 super(Py2X, self).__init__(*args, **kwargs)
1965
1966 ## String identifier of current instance.
1967 self.MakeObjectKind = 'Py2X'
1968
1969 ## Define whether Intel's MKL should be statically or dynamically linked.
1970 # Defaults to True, meaning that Intel's MKL has to be provided by the operating system.
1971 self.no_static_mkl = kwargs.get('no_static_mkl', True)
1972
1973 ## Define whether Intel's MKL should be discarded
1974 # Defaults to False on NT systems. Defaults to True on Linux systems in a Docker instance. Overwrites previous setting.
1975 self.no_mkl = kwargs.get("no_mkl", self.hasFosshasFoss and (Utility.GetExecutable("choco") or Utility.GetPlatform() in ["linux","cygwin","msys","darwin"]))
1976
1977 ## Define whether the architecture shall be appended to the build name.
1978 # Defaults to False, meaning that the architecture is appended.
1979 self.no_append_arch = kwargs.get('no_append_arch', False)
1980
1981 ## Define if the input should be compiled exactly as provided.
1982 # Defaults to False, meaning that merging & pre-processing utilities will be carried out.
1983 self.incrementalincremental = kwargs.get('incremental', False)
1984
1985 # Immutable settings for Py2X object
1986 ## Absolute system path to Python executable.
1987 self.path2exe = sys.executable.replace("\python.exe","")
1988 ## The executable command used in the main build event.
1989 self.exe = 'python.exe'
1990
1991 # Interoperability change. Always put executables in quotes (who would have guessed...)
1992 self.exe = '"{}" '.format(os.path.join(self.path2exe,self.exe))
1993
1994 # Get valid f2py call.
1995 old_numpy_call = os.path.join(self.path2exe, "Scripts","f2py.py")
1996 new_numpy_call = os.path.join(self.path2exe, "Scripts","f2py.exe")
1997
1998 if os.path.exists(old_numpy_call):
1999 self.exe += '"{}"'.format(old_numpy_call)
2000 os.environ["pyx_compiler"] = " ".join([sys.executable,old_numpy_call])
2001 elif os.path.exists(old_numpy_call):
2002 self.exe = '"{}"'.format(new_numpy_call)
2003 os.environ["pyx_compiler"] = new_numpy_call
2004 else:
2005 ## Verify that a f2py version is accessible
2006 from numpy import f2py
2007 ## We are either on Linux or within a virtual environment (Poetry). Attempt to find f2py through the
2008 # $PATH variable. Raise an error to indicate that the operation will not succeed.
2009 os.environ.update({"PATH":os.pathsep.join([os.getenv("PATH")]+
2010 [os.path.join(f2py.__path__[0].split(os.path.sep+"lib")[0],"bin")]+
2011 [os.path.join(os.path.dirname(x),"Scripts") if Utility.GetPlatform() in ["windows"] else
2012 os.path.join(os.path.dirname(x),"bin") for x in sys.path if x.endswith("lib")])})
2013 # Find f2py in global PATH variable
2014 found, self.exe = Utility.GetExecutable("f2py", get_path=True)
2015 # Raise a runtime error if unsuccessful
2016 if not found: raise RuntimeError("f2py could not be found on the system.")
2017 # Otherwise, use the absolute path
2018 os.environ["pyx_compiler"] = self.exe
2019 self.exe = '"{}"'.format(self.exe)
2020 self.path2exe = ""
2021
2022 ## Temporary build name of current job.
2024
2025 if self.architecturearchitecture == "x86":
2026 mkl_interface_lib = "mkl_intel_c"
2027 else:
2028 mkl_interface_lib = "mkl_intel_lp64"
2029
2030 if self.no_mkl:
2031 # Remove include files from MKL explicitly if requested
2032 if version.parse(np.__version__) >= version.parse("1.26.0"):
2033 for x in copy.deepcopy(self.incdirsincdirs):
2034 if "mkl" in x: self.incdirsincdirs.remove(x)
2035 # Do nothing
2036 else: pass
2037 elif not self.no_static_mkl:
2038 # Link statically against MKL library. Deactivate this option by default.
2039 self.libslibslibs.append([mkl_interface_lib, "mkl_intel_thread", "mkl_core", "libiomp5md"])
2040 self.libslibslibs = list(Utility.ArbitraryFlattening(self.libslibslibs))
2041 else:
2042 # Provide additional default dependencies (required for MCODAC).
2043 if Utility.IsDockerContainer() and Utility.GetPlatform() not in ["windows", "msys"]: self.libslibslibs.append(["gomp","ifcore"])
2044 # Link dynamically against MKL library.
2045 self.libslibslibs.append("mkl_rt")
2046 self.libslibslibs = list(Utility.ArbitraryFlattening(self.libslibslibs))
2047
2048 # MKL is requested. Update library path to include local pip dependencies.
2049 if not self.no_mkl or not self.no_static_mkl or any(["mkl" in x for x in self.libslibslibs]):
2050 # Always add local path to library search path.
2051 prefix = "Library" if Utility.GetPlatform() in ["windows"] else ""
2052 # Catch exception while running in a local virtual environment
2053 base = os.path.dirname(os.path.dirname(os.getenv("pyx_compiler", sys.prefix)))
2054 # Update library search paths
2055 self.libdirslibdirs.extend([os.path.join(x,prefix,"lib") for x in (sys.prefix, base)])
2056
2057 if self.no_append_arch:
2058 self.architecturearchitecture = ''
2059
2060 # Immutable settings for Py2X object
2061 if self.incrementalincremental:
2062 c_files = [x for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]
2063 self.exe += ' -c -m %s %s' % (self.buildidbuildid+self.architecturearchitecture, ' '.join(c_files))
2064 else:
2065 self.exe += ' -c -m %s %s' % (self.buildidbuildid+self.architecturearchitecture, self.buildnamebuildname)
2066
2067 # Copy default mapping file to scratch directory (eq. workspace if not specified)
2068 if not self.barebare: copyfile(os.path.join(Path2Config,".f2py_f2cmap"), os.path.join(self.scrtdirscrtdir,".f2py_f2cmap"))
2069
2070 # Strip path from executable if already present.
2071 if self.exe.startswith(self.path2exe,1):
2072 self.path2exe = ""
2073
2074 ## Tuple of temporary files deleted after job completion. Has already stored custom variable declaration
2075 # mapping file used by f2py.
2076 self.tempstempstemps = self.tempstempstemps + (".f2py_f2cmap", self.buildnamebuildname)
2077
2078 ## Iterate through all active processes matching the current BuildID
2079 # and kill them. Do not attempt on public machines (kills everything silently).
2080 if Utility.IsDockerContainer() and not self.barebare:
2081 for proc in psutil.process_iter():
2082 if os.getpid() == proc.pid:
2083 continue # Skip self.
2084 try:
2085 for key, value in proc.as_dict().items():
2086 if (str(self.buildidbuildid) in str(key) or str(self.buildidbuildid) in str(value)) and Utility.IsNotEmpty(self.buildidbuildid):
2087 # Get process name & PID from process object. Kill the process and write a message.
2088 proc_delimn = " "; processName = proc.name(); processID = proc.pid; #proc.kill()
2089 print("==================================")
2090 print("Found existing child process @ %s" % proc_delimn.join([str(processName),':::',str(processID)]))
2091 print("The process is aborted for compilation")
2092 print("==================================")
2093 except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
2094 pass
2095
2096 # Skip evaluation after one successful run.
2097 if not Utility.IsNotEmpty(self.buildidbuildid): break
2098
2099 # Identify source code and BuildID and set the corresponding environment variables for Mingw64 and Linux.
2100 if kwargs.get("bash", self.hasFosshasFoss):
2101 try:
2102 os.environ["pyx_buildid"], os.environ["pyx_source"] = (self.buildidbuildid+self.architecturearchitecture, ' '.join(c_files))
2103 except:
2104 os.environ["pyx_buildid"], os.environ["pyx_source"] = (self.buildidbuildid+self.architecturearchitecture, self.buildnamebuildname)
2105 pass
2106
2107 @staticmethod
2108 def inspect(package,**kwargs): # pragma: no cover
2109 """
2110 Inspect the content of a given f2py package. Returns all qualified modules with their respective functions
2111 """
2112 # Create a modifiable condition
2113 def condition(x): return kwargs.get("custom_condition",not x.startswith("_") and not x.endswith("__"))
2114 # Collect all modules
2115 all_modules = [x for x in dir(package) if condition(x)]
2116 # Return all functions associated with a given module.
2117 return [".".join([package.__name__,x,y]) for x in all_modules for y in dir(getattr(package, x)) if condition(y)]
2118
2119 @staticmethod
2120 def show(package, feature, **kwargs): # pragma: no cover
2121 """
2122 Inspect the documentation content of a given f2py package feature by default.
2123 Optional, define callback to return any other existing attribute.
2124 """
2125 import functools
2126 delimn = "."; anchor = feature.split(".");
2127 # Helper functions
2128 def rgetattr(obj, attr, *args):
2129 """
2130 Get an attribute from a nested object.
2131
2132 @note: https://stackoverflow.com/questions/31174295/getattr-and-setattr-on-nested-subobjects-chained-properties
2133 """
2134 def _getattr(obj, attr):
2135 """
2136 Wrapper of existing getattr function
2137 """
2138 try: result = getattr(obj, attr, *args)
2139 except:
2140 relative = "."*(anchor.index(str(attr))-1)+str(attr)
2141 result = importlib.import_module(relative, package=".".join(anchor[0:relative.count(".")+1]))
2142 return result
2143 return functools.reduce(_getattr, [obj] + attr.split('.'))
2144 # Import the given module
2145 module = importlib.import_module(package.__name__)
2146 # Collect the requested attribute. Remove the package name itself
2147 attributes = delimn.join([x for x in feature.split(delimn) if x not in [package.__name__]])
2148 # Return the attributes docstring
2149 return getattr(rgetattr(module, attributes),kwargs.get("callback","__doc__")) if (attributes and attributes not in [delimn]) else getattr(module,kwargs.get("callback","__doc__"))
2150
2151 @staticmethod
2152 def callback(*args): # pragma: no cover
2153 """
2154 Get callback of any Python object.
2155 """
2156 # Access callback of any Python object directly.
2157 return Py2X.show(*args,callback="__call__")
2158
2159 @classmethod
2160 def parse(cls, *args, **kwargs): # pragma: no cover
2161 """
2162 Execute the current class as a CLI command.
2163 """
2164 # Import its main from VTL
2165 from PyXMake.VTL import py2x
2166 # Evaluate current command line
2167 command = kwargs.get("command",sys.argv)
2168 # Process all known arguments
2169 parser = argparse.ArgumentParser(description="Build a shared Fortran library for current Python executable")
2170 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
2171 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
2172 parser.add_argument('-f', '--files', nargs='+', default=[], help="Source file or list of all source files in the order of compilation")
2173 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
2174 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional directories and files required for the build.")
2175 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for the build. Defaults to current workspace.")
2176 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
2177 parser.add_argument("--format", type=str, nargs=1, help="Toggle between fixed and free format code style. Defaults to Fixed.")
2178 parser.add_argument("--incremental", type=Utility.GetBoolean, const=True, default=False, nargs='?',
2179 help="Toggle between incremental and non-incremental build. Defaults to False.")
2180 parser.add_argument("--foss", type=Utility.GetBoolean, const=True, default=True, nargs='?',
2181 help="Toggle to enforce free-open source builds. Defaults to True.")
2182 # Check all options or run unit tests in default mode
2183 try:
2184 # Check CLI options
2185 _ = command[1]
2186 args, _ = parser.parse_known_args(command[1:])
2187 # Project name is mandatory
2188 project = args.name[0]
2189 # Specification of source directory is mandatory
2190 source = args.source[0] ;
2191 # Optional non-default output directory
2192 try: files = args.files
2193 except: files = []
2194 # Optional non-default output directory
2195 try: output = args.output[0]
2196 except: output = os.path.abspath(os.getcwd())
2197 # Optional non-default scratch directory
2198 try: scratch = args.scratch[0]
2199 except: scratch = os.path.abspath(os.getcwd())
2200 # Optional non-default definition of additional include directories
2201 try:
2202 _ = args.include[0]
2203 # Collect all given paths. Get system independent format
2204 include = Utility.GetSanitizedDataFromCommand(args.include)
2205 # No extra test cases have been given
2206 except: include = []
2207 # Optional code format style definition. Defaults to Fixed.
2208 try: fformat = args.format[0]
2209 except: fformat = "fixed"
2210 # Optional incremental build option. Defaults to False.
2211 try: incremental = args.incremental[0]
2212 except: incremental = False
2213 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
2214 try: verbosity = int(args.verbosity[0])
2215 except: verbosity = 0
2216 # Optional non-default package check
2217 try: foss = args.check[0]
2218 except: foss = True
2219 # Create a dictionary combining all settings
2220 settings = {"files":files, "incremental": incremental, "include": include,
2221 "source":source, "output":output, "scratch":scratch,
2222 "format":fformat, "foss":foss,
2223 "verbosity":verbosity}
2224 # Use an exception to allow help message to be printed.
2225 except Exception as _:
2226 # Local imports. These are only meaningful while executing an unit test
2227 from PyXMake import VTL
2228 # Build all supported features for current version (default options)
2229 if AllowDefaultMakeOption:
2230 # Predefined script local variables
2231 __arch = Utility.GetArchitecture()
2232 __foss = Utility.IsDockerContainer() or Utility.GetPlatform() in ["linux"]
2233 try:
2234 # Import PyCODAC to build library locally during setup.
2235 from PyCODAC.Tools.Utility import GetPyCODACPath
2236 # Import and set local path to PyCODAC
2237 __mcd_core_path = os.path.join(GetPyCODACPath(),"Core")
2238 except ImportError:
2239 # This script is not executed as plug-in
2240 __mcd_core_path = ""
2241 except:
2242 # Something else went wrong.
2243 from PyXMake.Tools import ErrorHandling
2244 ErrorHandling.InputError(20)
2245 # Build BoxBeam
2246 BuildID = 'bbeam'
2247 py2x(BuildID,
2248 files=VTL.GetSourceCode(1),
2249 source=os.path.join(__mcd_core_path,"external","boxbeam"),
2250 libs=[],include=[],dependency=[],verbosity=2,
2251 # BoxBeam binary is referenced in PyOCDAC. Updated is performed there
2252 output=os.path.join(os.path.join(__mcd_core_path,"bin",Utility.GetPlatform(),__arch)), foss=__foss)
2253 # Build Beos
2254 BuildID = 'beos'
2255 py2x(BuildID,
2256 files=VTL.GetSourceCode(2),
2257 source=os.path.join(__mcd_core_path,"external","beos"),
2258 libs=[],include=[],dependency=[],
2259 # Beos binary is referenced in PyOCDAC. Updated is performed there
2260 output=os.path.join(os.path.join(__mcd_core_path,"bin",Utility.GetPlatform(),__arch)), foss=__foss)
2261 # Build MCODAC (default settings)
2262 BuildID = "mcd_core"; py2x(BuildID, foss=__foss)
2263 else:
2264 fformat = settings.pop("format")
2265 # Execute CLI command
2266 py2x(project, command=VTL.GetBuildCommand(0, _format=fformat), **settings)
2267
2268## @class PyXMake.Build.Make.PyInstaller
2269# Base class for all PyInstaller build events. Inherited from Make.
2270class PyInstaller(Make):
2271 """
2272 Inherited class to build projects using PyInstaller.
2273 """
2274 def __init__(self, *args, **kwargs):
2275 """
2276 Initialization of PyInstaller class object.
2277
2278 @note Creates stand-alone application of Python scripts using PyInstaller.
2279 """
2280 super(PyInstaller, self).__init__(*args, **kwargs)
2281 ## String identifier of current instance.
2282 self.MakeObjectKind = "PyInstaller"
2283
2284 # Set build mode
2285 self.buildtype = kwargs.get("type","onefile")
2286
2287 # Remove all default libraries, paths and includes from Make class.
2288 self.libslibs = []
2289 self.libdirslibdirs = []
2290 self.incdirsincdirs = []
2291
2292 # Add specification file to temporaries
2293 self.tempstemps += (self.buildid+".spec",)
2294 pass
2295
2296 def Encryption(self, encrypt, **kwargs):
2297 """
2298 Encrypt byte-code by using user-supplied or randomly generated key.
2299
2300 @author: garb_ma
2301 @param encrypt: Boolean
2302 """
2303 if encrypt:
2304 self.key_string = kwargs.get("key_string",''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16)))
2305 return
2306
2307 def Preprocessing(self, cmdstring=''):
2308 """
2309 Assemble command string for the pre-build event.
2310 """
2311 delimn = " "
2312 # Store command string in object
2313 ## Command executed during pre-build event.
2314 self.precmdprecmd = delimn.join([self.iniCompiler,cmdstring])
2315 pass
2316
2317 def Build(self, mode="onefile", **kwargs):
2318 """
2319 Switch build mode. Defaults to one single file.
2320 """
2321 import py_compile
2322 # Get shorthands for compatibility modes
2323 try:
2324 from PyInstaller.compat import is_py27 #@UnresolvedImport
2325 except:
2326 is_py27 = sys.version_info[0] == 2
2327 finally:
2328 from PyInstaller.compat import is_py35, is_py36, is_py37 #@UnresolvedImport
2329
2330 # Parse relevant versions from packages
2331 from setuptools import __version__ as set_version
2332 from PyInstaller import __version__ as pyi_version
2333
2334 # Set build mode
2335 self.buildtype = mode
2336
2337 # Reset command
2338 self.makecmdmakecmdmakecmd = ""
2339 # Create base command and add additional options based on identified inputs.
2340 self.makecmdmakecmdmakecmd = ['--name=%s' % self.buildid,'--%s' % self.buildtype, '--clean', "--noconfirm"]#, "--windowed"]
2341
2342 # Customize the application icon
2343 self.makecmdmakecmdmakecmd.append("--icon=%s" % kwargs.get("icon",os.path.join(Path2Config,"stm_logo.ico")))
2344
2345 # Correction of required missing import for new releases of setuptools. Deprecated since major release 49
2346 if version.parse("45.0.0") < version.parse(set_version) < version.parse("49.0.0"): # pragma: no cover
2347 self.makecmdmakecmdmakecmd.insert(1,'--hidden-import=%s' % "pkg_resources.py2_warn")
2348 # Adding request library explicitly.
2349 if version.parse(set_version) >= version.parse("64.0.0"): self.makecmdmakecmdmakecmd.insert(1,'--hidden-import=%s' % "requests")
2350
2351 # Use UPX compression to create a smaller application folder/file
2352 if Utility.GetPlatform() in ["windows"]: # pragma: no cover
2353 # Attempt to find UPX from global installation path
2354 if Utility.GetExecutable("upx"): self.makecmdmakecmdmakecmd.append("--upx-dir=%s" % Utility.GetExecutable("upx", get_path=True)[-1])
2355 # Fetch supplied executable from path (deprecated, path will not exists in the future)
2356 elif os.path.exists(os.path.join(PyXMakePath,"Build","bin","upx")): self.makecmdmakecmdmakecmd.append("--upx-dir=%s" % os.path.join(PyXMakePath,"Build","bin","upx"))
2357
2358 # Some binaries are unusable after the obfuscation process. We skip those here.
2359 if is_py27:
2360 excludes = ["python2.dll","python27.dll", "qwindows.dll"]
2361 elif is_py35:
2362 excludes = [x for x in os.listdir(os.path.dirname(sys.executable)) if x.endswith((".dll"))]
2363 elif is_py36:
2364 excludes = [x for x in os.listdir(os.path.dirname(sys.executable)) if x.endswith((".dll"))]
2365 elif is_py37:
2366 excludes = ["python3.dll","python37.dll","ucrtbase.dll", "qwindows.dll"]
2367
2368 ## Remove the complete scipy package from UPX shrinking
2369 # 28.06.2021 // garb_ma
2370 try:
2371 import scipy
2372 # Add scipy exceptions
2373 if version.parse(scipy.__version__) <= version.parse("1.6.3"): excludes.extend(Utility.ArbitraryFlattening([x[-1] for x in os.walk(scipy.__path__[0])]))
2374 except: pass
2375
2376 upx_exclude = ["--upx-exclude=%s" % x for x in excludes]
2377 self.makecmdmakecmdmakecmd.extend(upx_exclude)
2378 # Do not execute image size reduction using UPX
2379 else: self.makecmdmakecmdmakecmd.extend(["--noupx"])
2380
2381 # Mark additional Python files to be (re-compiled) before copying directly into the executable.
2382 add_compile = list(Utility.ArbitraryFlattening(kwargs.get("add_compile",[])))
2383
2384 # Add additional include directories
2385 if hasattr(self, "incdirs"): # pragma: no cover
2386 for data in self.incdirsincdirs:
2387 if os.path.exists(data.split(os.path.pathsep)[0]):
2388 new_path = data.split(os.path.pathsep)[0]
2389 # Compile source code files to byte-code
2390 if Utility.PathLeaf(new_path).endswith(".py") and Utility.PathLeaf(new_path) in add_compile:
2391 py_compile.compile(new_path,os.path.join(self.scrtdirscrtdir,Utility.PathLeaf(new_path)+"c"))
2392 self.tempstemps = self.tempstemps + (Utility.PathLeaf(new_path)+"c",)
2393 new_path = os.path.join(self.scrtdirscrtdir,Utility.PathLeaf(new_path)+"c")
2394 data = os.path.pathsep.join([new_path,data.split(os.path.pathsep)[1]])
2395 self.makecmdmakecmdmakecmd.insert(1,'--add-data=%s' % data)
2396 else:
2397 for path in sys.path:
2398 new_path = os.path.join(path,data.split(os.path.pathsep)[0])
2399 if os.path.exists(new_path) and os.path.isdir(new_path):
2400 # Check if content of folders should be presented in plain text.
2401 if not kwargs.get("add_plain",False):
2402 for root, _, files in Utility.PathWalk(new_path):
2403 # Ignore repository folders (no exceptions!)
2404 if any(x in root for x in [".git", ".svn"]):
2405 continue
2406 for file in files:
2407 # Ignore files matching specified patterns
2408 if not file.endswith((".pdf")) and not any(x in file for x in [".cpython"]):
2409 base_path = data.split(os.path.pathsep)[-1]; sub_path = root.split(base_path)[-1]
2410 sub_struct =os.path.join(base_path,sub_path.lstrip(os.path.sep),".")
2411
2412 if len(data.split(os.path.pathsep)) != 1:
2413 save_path = os.path.join(data.split(os.path.pathsep)[-1],".")
2414 else:
2415 save_path = sub_struct
2416 # Skip python files if compiled scripts are available
2417 if file.endswith(".py") and os.path.exists(os.path.join(new_path,file+"c")):
2418 continue
2419 elif file.endswith(".py") and file in add_compile:
2420 # Compile all blank source files to byte-code to obfuscate the code.
2421 py_compile.compile(os.path.join(new_path.split(data.split(os.path.pathsep)[0])[0],sub_struct.rstrip("."),file),
2422 os.path.join(new_path.split(data.split(os.path.pathsep)[0])[0],sub_struct.rstrip("."),file+"c"))
2423 file += "c"
2424 # Loop over all files within each directory and store them appropriately
2425 self.makecmdmakecmdmakecmd.insert(1,'--add-data=%s' % os.path.pathsep.join([os.path.join(new_path.split(
2426 data.split(os.path.pathsep)[0])[0],sub_struct.rstrip("."),file),save_path]))
2427 else:
2428 # Add folder content as-is to the application
2429 self.makecmdmakecmdmakecmd.insert(1,'--add-data=%s' % os.path.pathsep.join([new_path,data.split(os.path.pathsep)[-1]]))
2430 break
2431 # Check if new directory is a file. Preserve original path, otherwise do nothing.
2432 elif os.path.exists(new_path) and os.path.isfile(new_path):
2433 if Utility.PathLeaf(new_path).endswith(".py") and Utility.PathLeaf(new_path) in add_compile:
2434 py_compile.compile(new_path,os.path.join(self.scrtdirscrtdir,Utility.PathLeaf(new_path)+"c"))
2435 self.tempstemps = self.tempstemps + (Utility.PathLeaf(new_path)+"c",)
2436 new_path = os.path.join(self.scrtdirscrtdir,Utility.PathLeaf(new_path)+"c")
2437 self.makecmdmakecmdmakecmd.insert(1,'--add-data=%s' % os.path.pathsep.join([new_path,os.path.join(os.path.dirname(data.split(os.path.pathsep)[-1]),".")]))
2438 break
2439 else:
2440 continue
2441
2442 # Set working (scratch) directory if being defined
2443 if hasattr(self, "scrtdir"):
2444 self.makecmdmakecmdmakecmd.insert(1,'--specpath=%s' % os.path.join(self.scrtdirscrtdir))
2445 self.makecmdmakecmdmakecmd.insert(1,'--workpath=%s' % os.path.join(self.scrtdirscrtdir,"build"))
2446
2447 # Set output path if being defined. Defaults to current working directory
2448 if hasattr(self, "outdir"):
2449 self.makecmdmakecmdmakecmd.insert(1,'--distpath=%s' % os.path.join(self.outdiroutdir))
2450
2451 # Add additional search paths
2452 if hasattr(self, "libdirs"):
2453 for path in self.libdirslibdirs:
2454 self.makecmdmakecmdmakecmd.insert(1,'--paths=%s' % path)
2455
2456 # Add encryption method. Deprecated in
2457 if hasattr(self, "key_string") and version.parse(pyi_version) <= version.parse("6.0.0"):
2458 self.makecmdmakecmdmakecmd.insert(1,'--key=%s' % self.key_string)
2459
2460 # Add source path (if given!)
2461 if hasattr(self, "srcdir") and self.srcdirsrcdir:
2462 self.makecmdmakecmdmakecmd.append(os.path.join(self.srcdirsrcdir,self.srcs[0]))
2463 else: # pragma: no cover
2464 self.makecmdmakecmdmakecmd.append(self.srcs[0])
2465
2466 # Set verbosity level
2467 if hasattr(self, "verbose"): # pragma: no cover
2468 if self.verboseverbose <= 0:
2469 level = "ERROR"
2470 elif self.verboseverbose == 1:
2471 level = "WARN"
2472 elif self.verboseverbose >= 2:
2473 level = "INFO"
2474 self.makecmdmakecmdmakecmd.insert(1,'--log-level=%s' % level)
2475
2476 @classmethod
2477 def parse(cls, **kwargs): # pragma: no cover
2478 """
2479 Execute the current class as a CLI command.
2480 """
2481 # Import its main from VTL
2482 from PyXMake.VTL import app
2483 # Evaluate current command line
2484 command = kwargs.get("command",sys.argv)
2485 parser = argparse.ArgumentParser(description="Build a stand-alone python application.")
2486 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
2487 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
2488 parser.add_argument('-f', '--file', nargs='+', default=[], help="Main entry point file.")
2489 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional files and folders required by the final executable.")
2490 parser.add_argument('-d', '--dependency', nargs='+', default=[], help="Additional search paths to resolve dependencies.")
2491 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
2492 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for the build. Defaults to current workspace.")
2493 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
2494 parser.add_argument("-m","--mode", type=str, nargs=1, default="onefile", help="Build type. Can be toggled between 'onefile' and 'onedir'.")
2495 parser.add_argument('--icon', type=str, nargs=1, help="Absolute path to a custom icon for the executable. Format: *.ico")
2496 parser.add_argument("--encryption", type=Utility.GetBoolean, const=True, default=False, nargs='?',
2497 help="Toggle between encryption and non-encryption build. Defaults to False.")
2498
2499 try:
2500 # Check CLI options
2501 _ = command[1]
2502 args, _ = parser.parse_known_args(command[1:])
2503 # Project name is mandatory
2504 project = args.name[0]
2505 # Specification of source directory is mandatory
2506 source = args.source[0] ;
2507 # Optional non-default output directory
2508 try: file = args.file
2509 except: file = []
2510 # Optional non-default output directory
2511 try: output = args.output[0]
2512 except: output = os.path.abspath(os.getcwd())
2513 # Optional incremental build option. Defaults to False.
2514 try: encryption = args.encryption[0]
2515 except: encryption = False
2516 # Optional non-default scratch directory
2517 try: scratch = args.scratch[0]
2518 except: scratch = os.path.abspath(os.getcwd())
2519 # Optional non-default additional include files and directories
2520 try: include = args.include[0]
2521 except: include = []
2522 # Optional non-default dependencies
2523 try: dependency= args.dependency[0]
2524 except: dependency = []
2525 # Optional non default build mode. Defaults to bundled one file.
2526 try: mode = args.mode[0]
2527 except: mode = "onefile"
2528 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
2529 try: verbosity = int(args.verbosity[0])
2530 except: verbosity = 0
2531 # Optional user-defined icon. If left blank, a default icon is used.
2532 try: icon = str(args.icon[0])
2533 except: icon = None
2534 # Create a dictionary combining all settings
2535 settings = {"script":file,
2536 "source":source, "output":output, "scratch":scratch,
2537 "mode":mode, "encryption": encryption,
2538 "verbosity":verbosity, "icon":icon}
2539 # Only parse these settings when explicitly given
2540 if include: settings.update({"include":include})
2541 if dependency: settings.update({"dependency":dependency})
2542 # Use an exception to allow help message to be printed.
2543 except Exception as _:
2544 BuildID = "stmlab"
2545 # Build all supported features for current Python version (default options)
2546 if AllowDefaultMakeOption: app(str(BuildID.lower()), mode="onedir")
2547 # Execute valid CLI command
2548 else: app(project, **settings)
2549 pass
2550
2551 def create(self,**kwargs):
2552 """
2553 Execute make command
2554 """
2555 # Import required package (only when function is actually executed)
2556 import PyInstaller.__main__ as pyi
2557
2558 # Increase recursion limit (required for OCC)
2559 sys.setrecursionlimit(kwargs.get("recursion_limit",int(os.getenv("pyx_recursion_limit",5000))))
2560
2561 # Go into scratch folder
2562 with Utility.ChangedWorkingDirectory(self.scrtdirscrtdir): # pragma: no cover
2563
2564 # Pre-build event (if required)
2565 try:
2566 if self.precmdprecmd != '':
2567 Utility.Popen(self.precmdprecmd, self.verboseverbose, collect=False)
2568 except:
2569 pass
2570
2571 # Create make command
2572 self.BuildBuild(mode=self.buildtype, **kwargs)
2573
2574 # Run build command
2575 pyi.run(self.makecmdmakecmdmakecmd)
2576
2577 # Finish and delete redundant files
2578 Utility.DeleteFilesbyEnding(self.tempstemps)
2579 shutil.rmtree('build', ignore_errors=True)
2580
2581 # End of class method
2582 return # pragma: no cover
2583
2584## @class PyXMake.Build.Make.NSIS
2585# Base class for all NSIS build events. Inherited from Make.
2586class NSIS(Make):
2587 """
2588 Inherited class to build projects using NSIS.
2589 """
2590 def __init__(self, *args, **kwargs):
2591 """
2592 Initialization of PyInstaller class object.
2593
2594 @note Creates a portable installer of a source folder using NSIS.
2595 """
2596 super(NSIS, self).__init__(*args, **kwargs)
2597 ## String identifier of current instance.
2598 self.MakeObjectKind = "NSIS"
2599
2600 # Remove all default libraries, paths and includes from Make class.
2601 self.libslibs = []
2602 self.libdirslibdirs = []
2603 self.incdirsincdirs = []
2604
2605 # Immutable settings for NSIS object
2606 ## Path to NSIS executable.
2607 self.path2exe = os.path.join(PyXMakePath,"Build","bin","nsis","App","NSIS")
2608 ## Executable of NSIS.
2609 self.exe = 'makensis.exe'
2610
2611 # Initialize build command
2612 if Utility.GetExecutable("makensis"): # pragma: no cover
2613 self.makecmdmakecmdmakecmd = Utility.GetExecutable("makensis", get_path=True)[-1]
2614 else: self.makecmdmakecmdmakecmd = os.path.join(self.path2exe,self.exe)
2615
2616 # Add specification file to temporaries
2617 self.tempstemps += (self.buildidbuildid+".nsh",)
2618 pass
2619
2620 def FTP(self, user, key, upload_file, host='ftp.dlr.de', path="/public/download/nightly"): # pragma: no cover
2621 """
2622 Define settings to establish a FTP connection.
2623 """
2624 import ftplib
2625 # Establish FTP connection to file sharing server
2626 ## Remote workspace. This is the upload directory for the given file
2627 self.ftp_client = ftplib.FTP(host, user, key)
2628 self.ftp_client.cwd(path)
2629 with open(upload_file, 'rb') as file:
2630 self.ftp_client.storbinary("STOR %s" % Utility.PathLeaf(upload_file), file)
2631 pass
2632
2633 @classmethod
2634 def parse(cls, **kwargs): # pragma: no cover
2635 """
2636 Execute the current class as a CLI command.
2637 """
2638 # Import its main from VTL
2639 from PyXMake.VTL import bundle
2640 # Evaluate current command line
2641 command = kwargs.get("command",sys.argv)
2642 # Process all known arguments
2643 parser = argparse.ArgumentParser(description="Create a portable distribution using a compressed archive or NSIS")
2644 parser.add_argument('name', type=str, nargs=1, help="Name of the resulting archive(s).")
2645 parser.add_argument('source', type=str, nargs=1, help="Absolute path to distribution folder.")
2646 parser.add_argument('-f', '--files', nargs='+', default=[], help="Source file or list of all source files used in the creation of the documentation.")
2647 parser.add_argument("-o", "--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current working directory.")
2648 parser.add_argument("-e","--exclude", nargs='+', default=[], help="File extensions to be ignored during the process")
2649 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for the build. Defaults to current workspace.")
2650 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
2651 parser.add_argument("--use_nsis", type=Utility.GetBoolean, const=True, default=False, nargs='?', help="Create an archive using NSIS. Defaults to False.")
2652 parser.add_argument('--extensions', nargs=argparse.REMAINDER,
2653 help="""
2654 All archive extensions. Defaults to tar.gz. List given as string separated by commas.
2655 Possible values are <zip>, <tar>, <tar.gz>, <bz2>.
2656 """)
2657
2658 # Command line separator
2659 delimn = ","
2660
2661 try:
2662 # Check CLI options
2663 _ = command[1]
2664 args, _ = parser.parse_known_args(command[1:])
2665 # Project name is mandatory
2666 project = args.name[0]
2667 # Check optional command line arguments
2668 source = args.source[0] ;
2669 # Optional non-default output directory
2670 try: files = args.files
2671 except: files = "."
2672 # Optional non-default output directory
2673 try: output = args.output[0]
2674 except: output = os.path.abspath(os.getcwd())
2675 # Optional non-default NSIS check. Defaults to False.
2676 try: check = args.use_nsis[0]
2677 except: check = False
2678 # Optional non-default scratch directory
2679 try: scratch = args.scratch[0]
2680 except: scratch = os.path.abspath(os.getcwd())
2681 # Optional non-default exclude pattern
2682 try: excludes = args.exclude[0]
2683 except: excludes = [".git", ".svn", "__pycache__"]
2684 # Optional non-default additional extensions
2685 try: extensions= args.extensions[0]
2686 except: extensions = delimn.join(["tar","tar.gz","tar.bz2","zip"])
2687 finally: extensions = extensions.split(delimn)
2688 # Create a dictionary combining all settings
2689 settings = {"source":source, "output":output, "extensions":extensions, "files":files,
2690 "scratch": scratch, "excludes":excludes, "use_nsis": check}
2691 # Use an exception to allow help message to be printed.
2692 except Exception as _:
2693 # Build all supported features
2694 if AllowDefaultMakeOption:
2695 # Create PyCODAC installer
2696 BuildID = "PyCODAC"; bundle(str(BuildID.lower()), use_nsis=Utility.GetExecutable("makensis"))
2697 # Execute valid CLI command
2698 else: bundle(project, **settings)
2699 pass
2700
2701 def create(self, **kwargs): # pragma: no cover
2702 """
2703 Execute make command
2704 """
2705 space = " "; point = "."; newline = '\n'
2706
2707 # Create bundle command script
2708 script = ["Unicode true"]
2709 script.append("!define MULTIUSER_EXECUTIONLEVEL user")
2710 script.append("!define ZIP2EXE_NAME `%s`" % self.buildidbuildid)
2711 script.append("!define ZIP2EXE_OUTFILE `%s`" % os.path.join(self.outdiroutdir,point.join([self.buildidbuildid,"exe"])))
2712 script.append("!define ZIP2EXE_COMPRESSOR_LZMA")
2713 script.append("!define ZIP2EXE_INSTALLDIR `$EXEDIR`")
2714 script.append("!include `${NSISDIR}\Contrib\zip2exe\Base.nsh`")
2715 script.append("!include `${NSISDIR}\Contrib\zip2exe\Modern.nsh`")
2716 if kwargs.get("install_path",""):
2717 script.append("InstallDir '%s'" % kwargs.get("install_path","$Desktop"))
2718 script.append("!insertmacro SECTION_BEGIN")
2719 script.append("SetOutPath $INSTDIR")
2720 # Add all source files to the bundle
2721 for src in self.srcssrcs:
2722 script.append("File /r /x *.nsh `%s`" % os.path.join(kwargs.get("assembly_path", self.srcdirsrcdir),src))
2723 script.append("!insertmacro SECTION_END")
2724 script.append('Icon "%s"' % kwargs.get("icon",os.path.join(Path2Config,"stm_logo.ico")))
2725 script.append("RequestExecutionLevel user")
2726
2727 # Go into scratch folder
2728 with Utility.ChangedWorkingDirectory(self.scrtdir if kwargs.get("use_nsis",True) else tempfile.gettempdir()):
2729 # Run build command
2730 MakeFile = Utility.GetTemporaryFileName(extension=".nsh")
2731
2732 # Populate script file with customizable features
2733 with open(MakeFile,"w") as f:
2734 f.write(newline.join(script))
2735
2736 # Execute command
2737 self.makecmdmakecmdmakecmd = space.join([self.makecmdmakecmdmakecmd, "/V3", os.path.join(self.scrtdir,MakeFile)])
2738
2739 ## Adding option to deactivate NSIS and instead use a compressed archive directly.
2740 # Useful on POSIX system w/o NSIS support
2741 if kwargs.get("use_nsis",True): Utility.Popen(self.makecmdmakecmdmakecmd, self.verboseverbose)
2742 else:
2743 for ext in kwargs.get("extensions",["tar.gz"]): Utility.CreateArchive(os.path.join(self.outdiroutdir,".".join([self.buildidbuildid,ext])),self.srcdirsrcdir, **kwargs)
2744
2745 # Add temporary file to tuple scheduled for removal
2746 self.tempstemps += (MakeFile,)
2747
2748 # Finish and delete redundant files
2749 Utility.DeleteFilesbyEnding(self.tempstemps)
2750
2751 # Upload results to a file sharing server
2752 if kwargs.get("upload",False) and not hasattr(self, "ftp_client"):
2753 user = kwargs.get("user",os.getenv("username","")); key = kwargs.get("key","")
2754 self.FTP(user, key, point.join([os.path.join(self.outdiroutdir,self.buildidbuildid),"exe"]))
2755
2756 # Success message
2757 print("==================================")
2758 print("Uploaded result to given FTP server")
2759 print("==================================")
2760
2761 # End of class method
2762 pass
2763
2764## @class PyXMake.Build.Make.Doxygen
2765# Base class for all Doxygen build events. Inherited from Make.
2766class Doxygen(Make):
2767 """
2768 Inherited class to automatically build a documentation using Doxygen.
2769 """
2770 def __init__(self, *args,**kwargs):
2771 """
2772 Initialization of doxygen class object.
2773 """
2774 super(Doxygen, self).__init__(*args,**kwargs)
2775 ## String identifier of current instance.
2776 self.MakeObjectKind = 'Doxygen'
2777
2778 os.environ['dox_pyjava'] = 'NO'
2779 os.environ['dox_fortran'] = 'NO'
2780 os.environ['dox_ccpp'] = 'NO'
2781 os.environ['dox_pdflatex'] = 'NO'
2782
2783 # Immutable settings for Doxygen object
2784 ## Path to Doxygen executable.
2785 self.path2exe = os.path.join(PyXMakePath,"Build","bin","doxygen")
2786 ## Executable of Doxygen.
2787 self.exe = 'doxygen.exe'
2788
2789 ## Redefine default delete command in dependence of platform
2790 remove = "del"
2791 # Use rm on Linux platforms.
2792 if Utility.GetPlatform() == "linux": remove = "rm -rf"
2793
2794 # Added Doxygen support for all systems.
2795 if Utility.GetExecutable("doxygen"):
2796 _, doxy_path = Utility.GetExecutable("doxygen",get_path=True)
2797 self.path2exe = os.path.dirname(doxy_path)
2798 self.exe = Utility.PathLeaf(doxy_path)
2799
2800 ## Update configuration file to the latest Doxygen syntax by default. Do not issue a warning (for now!)
2801 Utility.Popen(" ".join([os.path.join(self.path2exe,self.exe),"-u",os.path.join(Path2Config,"stm_doc_config")]),verbosity=0)
2802
2803 ## Type of source file. Can be one of Fortran, CCpp or Other.
2804 # Defaults to Fortran if not specified. Starts documentation procedure
2805 # for Java/Python if type is neither Fortran nor CCpp.
2806 if self.stypestype == 'Fortran':
2807 os.environ['dox_fortran'] = 'YES'
2808 ## Temporary build name of current job.
2809 self.buildnamebuildname = self.buildid+"_doc.f90"
2810 ## Tuple of temporary files scheduled for removal.
2812 elif self.stypestype == 'CCpp': # pragma: no cover
2813 os.environ['dox_ccpp'] = 'YES'
2814 self.buildnamebuildname = self.buildid+"_doc.cpp"
2815 self.tempstempstemps = self.tempstempstemps + (self.buildnamebuildname, )
2816 else:
2817 temp = []; delimn = " "
2818 self.buildnamebuildname = ''
2819 temp.append(self.srcssrcs)
2820 os.environ['dox_pyjava'] = 'YES'
2821 paths = list(Utility.ArbitraryFlattening(temp))
2822
2823 # Remove empty and/or unnecessary paths from joined input string
2824 for y in paths:
2825 for x in os.listdir(y):
2826 if x.endswith((".java", ".py")):
2827 if not any([z == "__init__.py" for z in os.listdir(y)]) and x.endswith((".py")): # pragma: no cover
2828 # Automatic documentation of files requires regular packages, not pure folders. Temporarily add an
2829 # __init__.py to every folder containing Python scripts. Removed after completion.
2830 open(os.path.join(y,"__init__.py"), 'a+').close()
2831 # Add absolute path to temporary files scheduled for removal.
2832 self.tempstempstemps = self.tempstempstemps + (os.path.join(y,"__init__.py"),)
2833 continue
2834 break
2835 else:
2836 paths.remove(y)
2837 if y in [os.path.dirname(k) for k in list(filter(os.path.isfile, self.tempstempstemps))]: # pragma: no cover
2838 paths.append(y)
2839
2840 self.buildnamebuildname = delimn.join(paths)
2841
2842 for x in list(filter(os.path.isfile, self.tempstempstemps)):
2843 while True: # pragma: no cover
2844 try:
2845 path = os.path.dirname(x)
2846 if os.path.exists(os.path.join(path,"__init__.py")):
2847 x = path
2848 else:
2849 break
2850 except:
2851 break
2852
2853 list(filter(os.path.isfile, self.tempstempstemps))
2854
2855 # Remove all directories from temporary array. We only seek files.
2856 self.tempstempstemps = tuple(filter(os.path.isfile, self.tempstempstemps))
2857
2858 # Check if non-directories exists
2859 if list(self.tempstempstemps): # pragma: no cover
2860 # Schedule them for removal
2861 rev_command = [remove] + list(self.tempstempstemps)
2862 self.postcmdpostcmd = delimn.join(rev_command)
2863
2864 # Initial environment variables (can be updated through the command line)
2865 os.environ['dox_input'] = self.buildnamebuildname
2866 os.environ["dox_version"] = str(1.0)
2867 os.environ['dox_images'] = Path2Config
2868 os.environ['dox_footer'] = os.path.join(Path2Config,"stm_doc_footer.html")
2869 os.environ['dox_logo'] = os.path.join(Path2Config,"stm_logo.png")
2870 os.environ['dox_pyfilter'] = (sys.executable+" "+os.path.join(Path2Config,"stm_pyfilter.py")) if Utility.GetPlatform() in ["windows","nt"] else (os.path.join(Path2Config,"stm_pyfilter.py"))
2871
2872 # Proper handling of Debian / macOS with Doxygen (latest version)
2873 if not sys.executable in os.getenv("PATH", "") and not Utility.GetPlatform() in ["windows","nt"]:
2874 os.environ['PATH'] = os.pathsep.join([sys.executable,os.getenv("PATH","")]);
2875 os.chmod(os.path.join(Path2Config,"stm_pyfilter.py"), stat.S_IWOTH); os.chmod(os.path.join(Path2Config,"stm_pyfilter.py"), stat.S_IXOTH)
2876
2877 # Set default color scheme.
2878 if not all([os.getenv(x) for x in ("dox_hue","dox_sat","dox_gamma")]):
2879 from PyXMake.Build.config.stm_color import DLRBlue as dox_color #@UnresolvedImport
2880 os.environ['dox_hue'], os.environ['dox_sat'], os.environ['dox_gamma'] = [str(int(round(x))) for x in np.multiply([360.,100.,100],
2881 np.array(colorsys.rgb_to_hsv(*(value/255 for value in
2882 ImageColor.getcolor(dox_color,"RGB")))))]
2883
2884 # Remove MKL from default command line
2885 ## Blank version of list containing library directories without initially specifying MKL.
2887 self.libdirslibdirs = []
2888 pass
2889
2890 def Settings(self, brief, header, outdir='', **kwargs):
2891 """
2892 Define environment variables for the default configuration file.
2893 """
2894 # Overwrite output directory
2895 if outdir != '':
2896 ## Output directory of current job. Defaults to current workspace if not given.
2897 self.outdiroutdir = outdir # pragma: no cover
2898
2899 # Set environment variables for configuration script
2900 os.environ['dox_brief'] = brief
2901 os.environ['dox_header'] = header
2902 os.environ['dox_outdir'] = self.outdiroutdir
2903
2904 # Update optional customization settings
2905 if kwargs.get("version",None): os.environ['dox_version'] = kwargs.get("version")
2906 if kwargs.get("icon",None):
2907 # Get the user defined file
2908 user_defined_image = kwargs.get("icon")
2909 # Create a new temporary file and fetch its file extension
2910 temporary_optimized_image =os.path.join(tempfile.mkdtemp(),"doxygen_icon%s" % os.path.splitext(user_defined_image)[-1])
2911 # Fetch optimal base width from default icon or refer to user input
2912 base_width = kwargs.get("base_width",Image.open(os.path.join(os.getenv("dox_images"),"stm_logo.png")).size[0])
2913 # Resize the image to its optimal proportions
2914 icon = Utility.GetResizedImage(user_defined_image, base_width)
2915 # Save the new image
2916 icon.save(temporary_optimized_image, optimize=True, quality=95, **icon.info)
2917 # Only add a custom logo as icon
2918 os.environ["dox_logo"], os.environ["dox_icon"] = (temporary_optimized_image,)*2
2919 # Set color scheme in HUE (HSV)
2920 if kwargs.get("color",""): # pragma: no cover
2921 # Only modify color scheme if a hex color was explicitly given
2922 os.environ['dox_hue'], os.environ['dox_sat'], os.environ['dox_gamma'] = [str(x) for x in np.multiply([360.,100.,100],
2923 np.array(colorsys.rgb_to_hsv(*(value/255 for value in
2924 ImageColor.getcolor(kwargs.get("color"),"RGB")))))]
2925 pass
2926
2927 @classmethod
2928 def parse(cls, **kwargs): # pragma: no cover
2929 """
2930 Execute the current class as a CLI command.
2931 """
2932 # Import its main from VTL
2933 from PyXMake.VTL import doxygen
2934 # Evaluate current command line
2935 command = kwargs.get("command",sys.argv)
2936 # Process all known arguments
2937 parser = argparse.ArgumentParser(description='CLI wrapper options for Doxygen with more sensible default settings.')
2938 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
2939 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
2940 parser.add_argument('-t', '--title', nargs='+', default=[], help="Header used in the documentation. If not given, defaults are created from the project name.")
2941 parser.add_argument('-f', '--files', nargs='+', default=[], help="Source file or list of all source files used in the creation of the documentation.")
2942 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for Doxygen. Defaults to current workspace.")
2943 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
2944 parser.add_argument("-c","--config", type=str, nargs=1, help="Absolute path to user-supplied Doxygen configuration file. Can be left blank.")
2945 parser.add_argument('-l', '--logo', nargs='+', default=[], help="Top level logo used by Doxygen. Defaults to structural mechanics department logo.")
2946 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
2947 parser.add_argument("-ta","--tag", type=str, nargs=1, help="Documentation version tag. Defaults to 1.0 if not set.")
2948 parser.add_argument("-fo","--format", type=str, nargs=1, help="Toggle between Java, Fortran and Python code base. Defaults to Fortran.")
2949 parser.add_argument("-fi","--filter", type=str, nargs=1, help="JSON configuration string for this operation")
2950 parser.add_argument('--icon', type=str, nargs=1, help=argparse.SUPPRESS)
2951 # Check all options or run unit tests in default mode
2952 try:
2953 # Check CLI options
2954 _ = command[1]
2955 args, _ = parser.parse_known_args(command[1:])
2956 # Project name is mandatory
2957 project = args.name[0];
2958 # Specification of source directory is mandatory
2959 source = args.source[0] ;
2960 # Optional non-default project title
2961 try:
2962 _ = args.title[0]
2963 header = [Utility.ArbitraryEval(x) for x in args.title]
2964 except: header = [project.title(),"%s Developer Guide" % project.title()]
2965 # Optional non-default output directory
2966 try: output = args.output[0]
2967 except: output = os.path.abspath(os.getcwd())
2968 # Optional non-default scratch directory
2969 try: scratch = args.scratch[0]
2970 except: scratch = os.path.abspath(os.getcwd())
2971 # Optional code format style definition. Defaults to Python.
2972 try: fformat = args.format[0]
2973 except: fformat = "Fortran"
2974 # Optional source code filter option
2975 scfilter = {"exclude": True, "startswith":(".","__")}
2976 # Sanitize user input
2977 try: scfilter.update(json.loads(Utility.ArbitraryEval(args.filter[0])))
2978 except: pass
2979 # Optional non-default source files.
2980 try:
2981 _ = args.files[0]
2982 files = [Utility.ArbitraryEval(x) for x in args.files]
2983 except:
2984 # Parse files directly if format equals Fortran source. Use folder structure in all other cases.
2985 index = 0 if fformat != "Fortran" else -1
2986 files = [x[index] for x in Utility.PathWalk(os.path.abspath(source), **scfilter)]
2987 # Non default user-supplied configuration file. Defaults to internal configuration file
2988 try: config = args.config[0];
2989 except: config = os.path.join(Path2Config,"stm_doc_config")
2990 # Optional non-default icon
2991 try: icon = args.logo[0]
2992 except: icon = getattr(args, "icon", [None] )
2993 if icon: icon = os.path.abspath(next(iter(icon)))
2994 # Optional version tag used for the documentation. Defaults to 1.0
2995 try: tag = args.tag[0]
2996 except: tag = str(1.0)
2997 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
2998 try: verbosity = int(args.verbosity[0])
2999 except: verbosity = 0
3000 # Create a dictionary combining all settings
3001 settings = {"title":header,
3002 "files":files,
3003 "ftype":fformat, "config":config,
3004 "source":source, "output":output, "scratch":scratch,
3005 "verbosity":verbosity,
3006 "icon":icon,"version":tag}
3007 # Use an exception to allow help message to be printed.
3008 except Exception as _:
3009 # Local imports. These are only meaningful while executing an unit test
3010 from PyXMake import VTL
3011 from PyCODAC.Tools.Utility import GetPyCODACPath
3012 # Build all supported features
3013 if AllowDefaultMakeOption:
3014 # Build documentation of PyXMake
3015 BuildID = "pyx_core"
3016 doxygen(BuildID, ["PyXMake", "PyXMake Developer Guide"],
3017 [x[0] for x in Utility.PathWalk(PyXMakePath, startswith=(".","__"), contains=("doc","bin","config"), endswith=("make","scratch","examples"))], "Python",
3018 output=os.path.join(PyXMakePath,"VTL","doc","pyx_core"))
3019 # Build documentation of PyCODAC
3020 BuildID = "pyc_core"
3021 doxygen(BuildID, ["PyCODAC", "PyCODAC Developer Guide"],
3022 [x[0] for x in Utility.PathWalk(GetPyCODACPath(), startswith=(".","__"),
3023 contains=("DELiS","Smetana","PyXMake","external","doc","cmd","bin","include","lib","config","fetch"),
3024 endswith=("VTL","make","scratch","examples","src","config","solver"))], "Python",
3025 output=os.path.join(GetPyCODACPath(),"VTL","doc","pyc_core"))
3026 # Build documentation of SubBuckling
3027 BuildID = "mcd_subbuck"
3028 doxygen(BuildID, ["SubBuck", "SubLaminate Buckling Developer Guide"],
3029 [x[0] for x in Utility.PathWalk(os.path.join(Utility.AsDrive("D"),"03_Workspaces","01_Eclipse","mcd_subbuckling"))], "Java",
3030 output=os.path.join(GetPyCODACPath(),"VTL","doc","mcd_subbuckling"))
3031 # Build documentation of Mapper
3032 BuildID = "mcd_mapper"
3033 doxygen(BuildID, ["Mapper", "Damage Mapping Developer Guide"],
3034 [x[0] for x in Utility.PathWalk(os.path.join(Utility.AsDrive("D"),"03_Workspaces","01_Eclipse","mcd_mapper"))], "Java",
3035 output=os.path.join(GetPyCODACPath(),"VTL","doc","mcd_mapper"))
3036 # Build documentation of BoxBeam
3037 BuildID = "box_core"
3038 doxygen(BuildID, ["BoxBeam", "BoxBeam Developer Guide"], VTL.GetSourceCode(1),
3039 source=os.path.join(os.path.join(GetPyCODACPath(),"Core"),"external","boxbeam"),
3040 output=os.path.join(GetPyCODACPath(),"Plugin","BoxBeam","VTL","doc","box_core"))
3041 # Build documentation of MCODAC (Default settings
3042 BuildID = 'mcd_core'
3043 doxygen(BuildID)
3044 else:
3045 # Execute CLI command
3046 doxygen(project, **settings)
3047 pass
3048
3049## @class PyXMake.Build.Make.Latex
3050# Base class for all Latex build events. Inherited from Make.
3051class Latex(Make):
3052 """
3053 Inherited class to automatically build a documentation using Latex
3054 """
3055 ## Non-default cross-version class property definition.
3056 # Protects a class property from unrestricted access
3057 class __property(object):
3058 def __init__(self, getter): self.getter= getter
3059 def __get__(self, instance, owner): return self.getter(owner)
3060
3061 # Definition of class attribute
3062 base_url = os.getenv("pyx_overleaf_url","https://overleaf.fa-services.intra.dlr.de")
3063
3064 # Adopt secret name in dependence of being distributed in a Docker container or not.
3065 secret = "pyc_overleaf_secret" if os.getenv("pyc_overleaf_secret","") else "pyx_overleaf_secret"
3066
3067 # Fail gracefully
3068 try: secret = open(os.getenv(secret,os.path.join(os.path.expanduser("~"),"Keys","Keepass","fa_overleaf_access")),"r").read()
3069 except: auth = {}
3070
3071 def __init__(self, *args, **kwargs):
3072 """
3073 Initialization of Latex class object.
3074 """
3075 # Allow for initialization w/o any source data
3076 if len(args) < 2: args += ("",)
3077 # Create super class from base class
3078 super(Latex, self).__init__(*args,**kwargs)
3079 ## String identifier of current instance.
3080 self.MakeObjectKind = 'Latex'
3081
3082 # Validate third party dependencies
3083 from six import exec_
3084 from PyXMake.Build import __install__ #@UnresolvedImport
3085 from sphinx import package_dir
3086
3087 # Check settings here for older python version compatibility
3088 silent_install = kwargs.get("silent_install",False)
3089
3090 ## Silently install all dependencies on-the-fly. Defaults to False, meaning Latex must be present in Path
3091 # or an external API service (like Overleaf) must be used.
3092 if silent_install: exec_(open(__install__.__file__).read(),{"__name__": "__main__", "__file__":__install__.__file__})
3093
3094 # Build script for Latex documentation
3095 os.environ["PATH"] += os.pathsep + os.pathsep.join([os.path.join(os.path.dirname(sys.executable),"Scripts")])
3096 # Add additional search paths to Latex default search paths
3097 os.environ["TEXINPUTS"] = os.getenv("TEXINPUTS", "")
3098 # Add custom SPHINX styles to Latex search path
3099 os.environ["TEXINPUTS"] += os.pathsep + os.pathsep.join([os.path.join(package_dir,"texinputs"),os.path.join(PyXMakePath,"Build","config")])
3100
3101 # Immutable settings for Latex object
3102 ## Path to Latex executable. Set using environment variable
3103 self.path2exe = ""
3104 ## Executable of Latex.
3105 self.exe = 'texify.exe'
3106
3107 ## Set default source and output directories for Latex build objects.
3108 # By default, both are set to the same folder containing the main source file.
3109 if os.path.exists(self.srcs[0]):
3110 self.srcdirsrcdir = os.path.dirname(os.path.abspath(self.srcs[0]))
3111 self.outdiroutdir = os.path.dirname(os.path.abspath(self.srcs[0]))
3112
3113 ## Set user-defined secret value, if given
3114 if kwargs.get("secret", None): Latex.secret = kwargs.get("secret")
3115
3116 # Remove MKL from default command line
3117 ## Blank version of list containing library directories without initially specifying MKL.
3119 self.libdirslibdirs = []
3120
3121 @__property
3122 def auth(self):
3123 """
3124 API access token defined as a protected class property for backwards compatibility.
3125 Works for both Python 2 and 3. Only meaningful when a remote Latex instance shall be called.
3126 """
3127 if not getattr(self, "_auth",{}):
3128 try:
3129 self._auth = {} ;
3130 result = self.session(*base64.b64decode(self.secret).decode('utf-8').split(":",1), base_url=self.base_url, use_cache=False) ;
3131 self._auth = result[-1] ;
3132 except: pass
3133 return self._auth
3134
3135 @classmethod
3136 def Settings(cls, **kwargs): # pragma: no cover
3137 """
3138 Define environment variables for the default configuration file.
3139 """
3140 # Set environment variables for configuration script
3141 os.environ["TEXINPUTS"] += os.pathsep + os.pathsep.join([str(value) for value in kwargs.values()])
3142 pass
3143
3144 @classmethod
3145 def session(cls, *args, **kwargs):
3146 """
3147 Create all required session tokens for an active Overleaf instance.
3148 """
3149 status_code = 500; result = {}
3150 # Use cached credentials. Defaults to True.
3151 if kwargs.get("use_cache",True): return [200,copy.deepcopy(getattr(cls, "auth",{}))]
3152 # Procedure
3153 try:
3154 from bs4 import BeautifulSoup
3155 # Create a new session
3156 session = requests.Session()
3157 email, password = args
3158 # Obtain login URL. Fail safe in case of empty input string.
3159 login_url = "{}/login".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3160 # Get secret token
3161 r = session.get(login_url); csrf = BeautifulSoup(r.text, 'html.parser').find('input', { 'name' : '_csrf' })['value']
3162 r = session.post(login_url, { '_csrf' : csrf , 'email' : email , 'password' : password })
3163 try: # pragma: no cover
3164 r.json(); status_code = 403;
3165 result = "The given credentials could not be verified."
3166 # Interrupt the process
3167 return [status_code, result]
3168 # Everything worked fine
3169 except: pass
3170 # Get valid CSRF Token for further commands
3171 r = session.get(login_url); csrf = BeautifulSoup(r.text, 'html.parser').find('input', { 'name' : '_csrf' })['value']
3172 # Fetch valid session token from the response header
3173 if r.status_code == 200:
3174 status_code = r.status_code; result = {"Cookie":r.headers.pop("Set-Cookie").split(ntpath.pathsep)[0],"_csrf":csrf}
3175 # Check if an error has occurred
3176 r.raise_for_status()
3177 # Error handler in top-level function
3178 except: pass
3179 # In all cases. Return something
3180 return [status_code, result]
3181
3182 @classmethod
3183 def show(cls,ProjectID, *args, **kwargs):
3184 """
3185 Show all build files from an Overleaf project remotely given its ID.
3186
3187 @note: Rebuilds the project in the process.
3188 """
3189 # Verify credentials
3190 code, result = cls.session(*args, **kwargs)
3191 if code != 200: return [code, result]
3192 # Update header
3193 header = result; header.update({'Content-Type':'application/json;charset=utf-8' })
3194 # Obtain project URL. Fail safe in case of empty input string.
3195 project_url = "{}/project".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3196 # Always build from scratch
3197 requests.delete(posixpath.join(project_url,ProjectID,"output"),headers={"X-Csrf-Token" if k == "_csrf" else k:v for k,v in result.items()})
3198 # Rebuild the project. Compiles everything.
3199 r = requests.post(posixpath.join(project_url,ProjectID,"compile"),params={"auto_compile":True}, headers=header,
3200 data=json.dumps({"check":"silent","incrementalCompilesEnabled":True,"_csrf":header.pop("_csrf")}))
3201 # Return the output file dictionary. Defaults to an empty dictionary in case something went wrong.
3202 if r.status_code == 200: return r.json()["outputFiles"]
3203 else: return {} # pragma: no cover
3204
3205 @classmethod
3206 def rename(cls, ProjectID, ProjectName, *args, **kwargs):
3207 """
3208 Rename an Overleaf project remotely given its ID.
3209 """
3210 # Verify credentials
3211 code, result = cls.session(*args, **kwargs)
3212 if code != 200: return [code, result]
3213 # Update header
3214 header = result; header.update({'Content-Type':'application/json;charset=utf-8' })
3215 # Fetch CSRF token from session result
3216 data = {"_csrf":result.pop("_csrf")}
3217 data.update({"newProjectName":str(ProjectName)})
3218 # Obtain project URL. Fail safe in case of empty input string.
3219 project_url = "{}/project".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3220 # Rebuild the project. Compiles everything.
3221 r = requests.post(posixpath.join(project_url,ProjectID,"rename"), headers=result, data=json.dumps(data))
3222 # Return the output file dictionary. Defaults to an empty dictionary in case something went wrong.
3223 if r.status_code == 200: return {"success":True,"message":r.text}
3224 else: return {} # pragma: no cover
3225
3226 @classmethod
3227 def upload(cls,archive, *args, **kwargs):
3228 """
3229 Upload a given archive to an active Overleaf instance creating a new project.
3230 """
3231 # Default value
3232 response = {}
3233 # Verify credentials
3234 code, result = cls.session(*args, **kwargs)
3235 if code != 200: return [code, result]
3236 # Obtain project upload URL. Fail safe in case of empty input string.
3237 upload_url = "{}/project/new/upload".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3238 filename = os.path.basename(archive)
3239 mime = "application/zip"
3240 files = {"qqfile": (filename, open(archive, "rb"), mime), "name": (None, filename)}
3241 params = {
3242 "_csrf":result.pop("_csrf"),
3243 "qquid": str(uuid.uuid4()),
3244 "qqfilename": filename,
3245 "qqtotalfilesize": os.path.getsize(archive),
3246 }
3247 # Execute the request
3248 r = requests.post(upload_url,params=params, files=files, headers=result)
3249 try:
3250 r.raise_for_status(); response = r.json()
3251 except Exception: pass
3252 # Check if JSON is valid.
3253 if not "success" in response: raise Exception("Uploading of %s failed." % archive )
3254 # Rename the newly created project
3255 cls.rename(r.json()["project_id"], str(filename.split(".")[0]).title(), *args, **kwargs)
3256 # Return response
3257 return response
3258
3259 @classmethod
3260 def download(cls, ProjectID, *args, **kwargs):
3261 """
3262 Download the complete archive or final pdf from an Overleaf project given its ID.
3263 """
3264 # Verify credentials
3265 code, result = cls.session(*args, **kwargs)
3266 if code != 200: return [code, result]
3267 # Default download path
3268 endpoint = posixpath.join("download","zip")
3269 # Download compiled PDF file instead
3270 if kwargs.get("output_format","zip") in ["pdf"]:
3271 # Verify that the source compiles with a new request
3272 endpoint = [x["url"] for x in cls.show(ProjectID, *args, **kwargs)
3273 if str(x["url"]).endswith("output.pdf")][0]
3274 ## Download in Overleaf has changed.
3275 # Refer to new download path if available but still support the deprecated alternative
3276 endpoint = next(iter([ posixpath.join("user",endpoint.split("user")[-1].replace(posixpath.sep,"",1))
3277 if "user" in endpoint.split(posixpath.sep)[:4]
3278 else posixpath.join("build",endpoint.split("build")[-1].replace(posixpath.sep,"",1)) ] ))
3279 # Obtain project URL. Fail safe in case of empty input string.
3280 project_url = "{}/project".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3281 # Rebuild the project. Compiles everything.
3282 r = requests.get(posixpath.join(project_url,ProjectID,endpoint), headers=result , stream=True)
3283 # Create the output file name
3284 output = os.path.abspath(".".join([os.path.join(os.getcwd(),ProjectID),endpoint[-3:]]))
3285 # Download the binary blob
3286 with open(output, "wb") as f:
3287 for chunk in r.iter_content(chunk_size=1024):
3288 if chunk: f.write(chunk)
3289 # Return the output path
3290 return output
3291
3292 @classmethod
3293 def delete(cls,ProjectID, *args, **kwargs):
3294 """
3295 Delete an Overleaf project given its ID.
3296 """
3297 # Check user settings. Defaults to False
3298 forever = kwargs.get("forever",False)
3299 # Verify credentials
3300 code, result = cls.session(*args, **kwargs)
3301 if code != 200: return [code, result]
3302 # Update header
3303 header = result; header.update({'Content-Type':'application/json;charset=utf-8' })
3304 # Obtain project URL. Fail safe in case of empty input string.
3305 project_url = "{}/project".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3306 # Execute the request
3307 r = requests.delete(posixpath.join(project_url,ProjectID), data=json.dumps({"forever": forever, "_csrf":header.pop("_csrf")}), headers=header)
3308 r.raise_for_status()
3309 # Return response
3310 return {"success":True,"message":r.text}
3311
3312 @classmethod
3313 def parse(cls, **kwargs): # pragma: no cover
3314 """
3315 Execute the current class as a CLI command.
3316 """
3317 # Import its main from VTL
3318 from PyXMake.VTL import latex
3319 # Evaluate current command line
3320 command = kwargs.pop("command",sys.argv)
3321 # Process all known arguments
3322 parser = argparse.ArgumentParser(description="Compile a TeXfile using MikTeX or Overleaf with custom templates.")
3323 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
3324 parser.add_argument('source', type=str, nargs=1, metavar="main.tex", help="Absolute path to the main source file. Must be either a Texfile or an archive.")
3325 parser.add_argument('-a', '--api', type=str, nargs=1, default="texworks", help="Additional files required for the process.")
3326 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional files required for the compilation process.")
3327 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
3328 # Check all options or run unit tests in default mode
3329 try:
3330 # Check CLI options
3331 _ = command[1]
3332 args, _ = parser.parse_known_args(command[1:])
3333 # Project name is mandatory
3334 project = args.name[0];
3335 # Specification of source directory is mandatory
3336 source = args.source[0] ;
3337 # Optional non-default compilation API
3338 try: config = args.api[0]
3339 except: config = "texworks"
3340 # Optional non-default scratch directory
3341 try: scratch = args.scratch[0]
3342 except: scratch = os.path.abspath(os.getcwd())
3343 # Optional non-default definition of additional files
3344 try:
3345 _ = args.include[0]
3346 # Collect all given paths. Get system independent format
3347 include = Utility.GetSanitizedDataFromCommand(args.include)
3348 # No extra files have been given
3349 except: include = []
3350 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
3351 try: verbosity = int(args.verbosity[0])
3352 except: verbosity = 2
3353 # Create a dictionary combining all settings
3354 settings = {"file":source, "API":config, "include":include, "scratch":scratch, "verbosity": verbosity}
3355
3356 # Use an exception to allow help message to be printed.
3357 except Exception as _:
3358 # Build all supported features
3359 if AllowDefaultMakeOption:
3360 # Run default option in a temporary directory
3361 with Utility.TemporaryDirectory():
3362 # Local import to make test work in a conda environment
3363 import PyXMake as _ #@UnusedImport
3364 # Use elsevier template for unit test
3365 from urllib.request import urlretrieve
3366 # Local variables
3367 BuildID = "elsevier"
3368 filename = "elsarticle.zip"
3369 url = "https://mirrors.ctan.org/macros/latex/contrib/elsarticle.zip"
3370 # Get a copy of keyword arguments
3371 settings = copy.deepcopy(kwargs)
3372 # Update test settings when CI specific environment variables can be found
3373 if os.getenv("CI_USER","") and os.getenv("CI_PASSWORD",""):
3374 settings.update({"secret": Utility.GetDockerEncoding(os.getenv("CI_USER"),os.getenv("CI_PASSWORD"))})
3375 # Down the template locally and run remote build process
3376 urlretrieve(url, filename); latex(BuildID, filename, API="Overleaf", **settings)
3377 else:
3378 # Execute CLI command
3379 latex(project, **settings)
3380 pass
3381
3382 def create(self, API="TeXworks", GUI=True, **kwargs):
3383 """
3384 Compile an existing project and/or start the graphical user interface.
3385 """
3386 command = []
3387 # Add all include directories to Latex environment variable
3388 os.environ["TEXINPUTS"] += os.pathsep + os.pathsep.join([str(include) for include in self.incdirsincdirs])
3389 ## Support both TeXWorks and Overleaf implementations
3390 # Use remote Overleaf implementation
3391 if API.lower() in ["overleaf"]:
3392 # Get Latex archive
3393 archive = os.path.abspath(self.srcs[0])
3394 # Operate fully on a temporary directory
3396 # Copy file into local scratch folder
3397 shutil.copy(archive, os.getcwd())
3398 # Upload the archive
3399 result = self.upload(archive)
3400 try:
3401 # If successful, retrieve the project id
3402 ProjectID = result["project_id"]
3403 # Download the resulting PDF file
3404 self.download(ProjectID, output_format="pdf")
3405 # Rename the project to build id
3406 self.rename(ProjectID, self.buildidbuildid)
3407 # Delete the project if not explicitly requested otherwise.
3408 if not kwargs.get("keep",False): self.delete(ProjectID)
3409 # Get name of the result file
3410 result_file = [x for x in os.listdir() if x.endswith(".pdf")][0]
3411 shutil.copy(result_file,self.outdiroutdir)
3412 # Explicitly request all errors for logging
3413 except Exception as e: print(e.args())
3414 pass
3415 # Use local TeXworks distribution
3416 elif API.lower() in ["texworks"]: # pragma: no cover
3417 if os.path.exists(self.srcs[0]):
3418 command.append(self.srcs[0])
3419 # Open GUI or run command directly from command line
3420 if GUI or not os.path.exists(self.srcs[0]):
3421 # Run GUI
3422 command.insert(0,"texworks.exe")
3423 subprocess.Popen(command, close_fds = True, **Make.Detach())
3424 else:
3425 with tempfile.TemporaryDirectory(dir=self.scrtdirscrtdir) as temp:
3426 # Set current directory to be the current directory
3427 os.chdir(temp)
3428 # Get name of output file
3429 output = os.path.splitext(Utility.PathLeaf(command[-1]))[0] + ".pdf"
3430 # Run command line
3431 command[0:0] = ["texify.exe","--pdf","--tex-option=-shell-escape","--synctex=1","--clean"]
3432 # Print output on-the-fly
3433 Utility.Popen(command, self.verboseverbose, collect=False)
3434 # Copy output file into the output directory
3435 shutil.copyfile(output, os.path.join(self.outdiroutdir,output))
3436 # Wait until copying has been completed
3437 os.chdir(self.scrtdirscrtdir)
3438 # Raise unknown API error
3439 else: raise NotImplementedError
3440 pass
3441
3442 @classmethod
3443 def __new(cls, ProjectName=None, *args, **kwargs): # pragma: no cover
3444 """
3445 Create a new project within the current session
3446 """
3447 # Check user settings. Defaults to False
3448 forever = kwargs.get("forever",False)
3449 # Verify credentials
3450 code, result = cls.session(*args, **kwargs)
3451 if code != 200: return [code, result]
3452 # Inherit BuildID from BuildID if given
3453 if not ProjectName: ProjectID = str(getattr(cls,"buildid",ProjectName))
3454 if not ProjectID: raise ValueError
3455 # Update header
3456 header = result; header.update({'Content-Type':'application/json;charset=utf-8' })
3457 # Obtain project URL. Fail safe in case of empty input string.
3458 project_url = "{}/project/new".format(kwargs.get("base_url", cls.base_url) or cls.base_url)
3459 # Create data segment
3460 data = {
3461 "_csrf": header.pop("_csrf"),
3462 "template": kwargs.get("template","template"),
3463 "projectName": ProjectID,
3464 "forever": forever,
3465 }
3466 # Execute the request
3467 r = requests.post(project_url, data=json.dumps(data), headers=header)
3468 r.raise_for_status()
3469 # Return response
3470 return {"success":True,"message":r.text,"response":r.json()}
3471
3472 # Abstract method. Can be invoked as an instance or class method.
3473 new = Utility.AbstractMethod(__new.__func__)
3474
3475## @class PyXMake.Build.Make.Coverage
3476# Base class for all Coverage build events. Inherited from Make.
3477class Coverage(Make): # pragma: no cover
3478 """
3479 Inherited class to automatically build a coverage report using Coverage.
3480 """
3481 def __init__(self, *args,**kwargs):
3482 """
3483 Initialization of coverage class object.
3484 """
3485 # Check if this class is used as a decorator.
3486 try: self.isDecorator = kwargs.pop("__as_decorator",any(line.startswith('@') for line in inspect.stack(context=2)[1].code_context))
3487 # Fail gracefully if context cannot be resolved.
3488 except TypeError: self.isDecorator = False
3489 # The current class is instantiated as a runner.
3490 if not self.isDecorator:
3491 super(Coverage, self).__init__(*args,**kwargs)
3492 ## String identifier of current instance.
3493 self.MakeObjectKind = 'Coverage'
3494 # Reset all pats to avoid collision
3495 self.incdirsincdirs = []; self.libdirs = []
3496 else:
3497 # Safe all arguments for later use.
3498 self.args = args
3499 self.kwargs = kwargs
3500
3501 def __call__(self, fn):
3502 """
3503 Implement an executable variant if class is used as a decorator.
3504 """
3505 import functools
3506 # Only run part of the code if environment is a test setup
3507 isPytest = self.kwargs.pop("autorun", False) or self.show()
3508 # Create a function wrapper
3509 @functools.wraps(fn)
3510 def decorated(*args, **kwargs):
3511 """
3512 Execute this part with given *args and **kwargs and the underlying function
3513 """
3514 # Execute test if running as a test
3515 if isPytest and not getattr(decorated,"has_run",False):
3516 compare = self.kwargs.pop("result",None);
3517 # Where running in a test environment
3518 try: result = fn(*self.args, **self.kwargs)
3519 except Exception as e: raise e
3520 ## Check if result matches the given value
3521 # Only meaningful when a result has been provided
3522 if compare: assert result == compare
3523 # Only run each test only once.
3524 decorated.has_run = True
3525 # Always return its true statement
3526 result = fn(*args, **kwargs)
3527 # Return the result of the function
3528 return result
3529 # Execute part directly when a function is loaded. Only in a test setup
3530 if isPytest:
3531 try: decorated()
3532 except TypeError: pass
3533 # Only execute the test setup once
3534 if not hasattr(decorated, "has_run"): decorated.has_run = False
3535 return decorated
3536
3537 @classmethod
3538 def show(cls):
3539 """
3540 Get a boolean value if this function is called within an active test environment
3541 """
3542 # Return the current root
3543 return str("pytest") in sys.modules or str("PYTEST_CURRENT_TEST") in os.environ
3544
3545 @classmethod
3546 def add(cls,*args, **kwargs):
3547 """
3548 Provide a test case for the given method as a decorator.
3549 """
3550 kwargs.update({"__as_decorator":True})
3551 # Do not return decorated function as a class in bundled mode
3552 if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
3553 def decorator(func): return func
3554 return decorator
3555 # Return current class instance
3556 else: return cls(*args,**kwargs)
3557
3558 @classmethod
3559 def parse(cls, **kwargs): # pragma: no cover
3560 """
3561 Execute the current class as a CLI command.
3562 """
3563 # Import its main from VTL
3564 from PyXMake.VTL import coverage
3565 # Evaluate current command line
3566 command = kwargs.get("command",sys.argv)
3567 # Process all known arguments
3568 parser = argparse.ArgumentParser(description='CLI wrapper options for Coverage with more sensible default settings.')
3569 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
3570 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
3571 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional files defining test cases required for test coverage.")
3572 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
3573 parser.add_argument("-e","--exclude", nargs='+', default=[], help="Full paths to be ignored from the coverage.")
3574 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
3575 # Check all options or run unit tests in default mode
3576 try:
3577 # Check CLI options
3578 _ = command[1]
3579 args, _ = parser.parse_known_args(command[1:])
3580 # Project name is mandatory
3581 project = args.name[0];
3582 # Specification of source directory is mandatory
3583 source = os.path.abspath(args.source[0]) ;
3584 # Optional non-default definition of additional tests cases
3585 try:
3586 _ = args.include[0]
3587 # Collect all given paths. Get system independent format
3588 include = Utility.GetSanitizedDataFromCommand(args.include)
3589 # No extra test cases have been given
3590 except: include = []
3591 # Optional non-default output directory
3592 try: output = args.output[0]
3593 except: output = os.path.abspath(os.getcwd())
3594 # Optional non-default exclude pattern
3595 try:
3596 _ = args.exclude[0]
3597 # Collect all given paths. Get system independent format
3598 exclude = Utility.GetSanitizedDataFromCommand(args.exclude)
3599 except: exclude = []
3600 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
3601 try: verbosity = int(args.verbosity[0])
3602 except: verbosity = 0
3603 # Verify that the given path is actually a package. Get the first entry
3604 source = [r for r, _, f in Utility.PathWalk(source) if any(x in ["__init__.py"] for x in f)][0]
3605 # Create a dictionary combining all settings
3606 settings = {"source":source, "output":output, "include":include,"exclude_paths":exclude,"verbosity":verbosity}
3607
3608 # Use an exception to allow help message to be printed.
3609 except Exception as _:
3610 # Run default test coverage of all integrated projects.
3611 if AllowDefaultMakeOption:
3612 # Run test coverage for PyXMake
3613 BuildID = "PyXMake"
3614 coverage(BuildID)
3615 else:
3616 # Execute CLI command
3617 coverage(project, **settings)
3618 pass
3619
3620 def Build(self, command=["--cov-fail-under=10","--cov-report=term-missing","--cov-report=html","--cov-report=xml","--junit-xml=junit.xml", "-W ignore::pytest.PytestAssertRewriteWarning"], **kwargs):
3621 """
3622 Assemble command strings for the main build event.
3623 """
3624 delimn = " "
3625 # Fetch additional make
3626 self.makecmdmakecmd = command
3627 if isinstance(self.makecmdmakecmd,str): self.makecmdmakecmd = self.makecmdmakecmd.split(delimn)
3628 pass
3629
3630 def create(self, **kwargs):
3631 """
3632 Execute make command
3633 """
3634 delimn = "_"
3635 # Copy the current command line
3636 sys_arg_recovery = copy.deepcopy(sys.argv);
3637 # Remove all trailing entries to avoid parsing them down.
3638 try: sys.argv[1:] = []
3639 except IndexError: pass
3640 # Copy the current system path
3641 sys_path_recovery = copy.deepcopy(sys.path)
3642 # Remote VTL directory from the overall system path to remove all shims.
3643 _ = [ sys.path.pop(i) for i, x in enumerate(sys.path) if x.endswith("VTL") ]
3644 # Check if dependencies can be resolved
3645 try: import pytest_cov as _ #@UnusedImport
3646 except: raise ImportError
3647 # Local imports
3648 import pytest
3649
3650 # Modify local VTL scratch directory
3651 from .. import PyXMakePath #@UnusedImport @Reimport
3652
3653 # Create make command if not already exists
3654 if not hasattr(self, "makecmd"): self.BuildBuild(**kwargs)
3655
3656 # Set default level of verbosity. Defaults to 1
3657 self.verboseverbose = kwargs.get("verbosity",1)
3658
3659 # Assemble make command
3660 command =self.makecmdmakecmd; command.extend(["--cov=%s" % path for path in self.srcssrcs])
3661
3662 # A ordered list of all supported configuration file formats
3663 config = kwargs.get("config",None)
3664 configfiles = ["pytest.ini","pyproject.toml","tox.ini","setup.cfg"]
3665
3666 # Check if a configuration file exists within the current directory
3667 if any(x in os.listdir(os.getcwd()) for x in configfiles) and not config:
3668 # Get the file
3669 for i, x in enumerate(configfiles):
3670 if os.path.exists(x): break
3671 # Overwrite configuration variable
3672 config = os.path.abspath(os.path.join(os.getcwd(),configfiles[i]))
3673
3674 # Default test directory
3675 test_directory = "tests"
3676
3677 # Operate fully in a temporary directory and deactivate user scratch space
3678 with Utility.TemporaryDirectory(), Utility.TemporaryEnvironment():
3679 os.mkdir(test_directory);
3680 # Fetch additional files
3681 [shutil.copy(x,os.path.join(os.getcwd(),test_directory,Utility.PathLeaf(x))) for x in self.incdirsincdirs if os.path.isfile(x)]
3682 # Create a test for each file honoring __main__
3683 for file in os.listdir(os.path.join(os.getcwd(),test_directory)):
3684 with open(os.path.join(test_directory,delimn.join(["test",Utility.PathLeaf(file)])),"w") as f:
3685 f.write("import runpy" +"\n")
3686 f.write('def test_script():'+"\n")
3687 f.write(" try: runpy.run_path('%s', run_name='__main__')" % str(os.path.join(test_directory,file)).replace(ntpath.sep,ntpath.sep*2) +"\n")
3688 f.write(" except SystemExit as exception: exitcode = exception.code" +"\n")
3689 f.write(" else: exitcode = 0" +"\n")
3690 # Copy all external tests
3691 [shutil.copy(os.path.join(self.srcdirsrcdir,x),os.path.join(os.getcwd(),test_directory)) for x in os.listdir(self.srcdirsrcdir)
3692 if os.path.isfile(os.path.join(self.srcdirsrcdir,x)) and x.startswith("test_")]
3693 # Create a default import check
3694 for path in self.srcssrcs:
3695 with open(os.path.join(test_directory,delimn.join(["test",Utility.PathLeaf(os.path.dirname(path)),".py"])),"w") as f:
3696 f.write("import os; import pkgutil;"+"\n")
3697 f.write('os.environ.update(%s)' % str(os.environ.copy()) +"\n")
3698 f.write('__all__ = [] ; __path__ = ["%s"];' % os.path.dirname(path).replace(ntpath.sep,ntpath.sep*2) +"\n")
3699 f.write('def test_import():'+"\n")
3700 f.write(' for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):'+"\n")
3701 f.write(' __all__.append(module_name); '+"\n")
3702 f.write(' _module = loader.find_module(module_name).load_module(module_name);'+"\n")
3703 f.write(' globals()[module_name] = _module'+"\n")
3704
3705 ## Always create a coverage setup file to prevent error during coverage collection
3706 with open(".coveragerc","w") as f:
3707 f.write("[run]"+"\n");
3708 # Remove given paths. Defaults to an empty list
3709 if kwargs.get("exclude_paths",[]): f.write("omit = "+"\n");
3710 for x in kwargs.get("exclude_paths",[]):
3711 # Update omit settings in dependence of the given path
3712 if os.path.isfile(x): path = str(x)
3713 else: path = str(x) + os.path.sep + "*"
3714 # Exclude all paths explicitly
3715 f.write(" *%s" % path +"\n");
3716 # Add default code exclude pattern
3717 f.write("[report]"+"\n");
3718 f.write("exclude_also ="+"\n");
3719 f.write(" def __repr__"+"\n");
3720 f.write(" if self.debug:"+"\n");
3721 f.write(" if settings.DEBUG"+"\n");
3722 f.write(" raise AssertionError"+"\n");
3723 f.write(" raise NotImplementedError"+"\n");
3724 f.write(" if 0:"+"\n");
3725 f.write(" except"+"\n");
3726 f.write(" if __name__ == .__main__.:"+"\n");
3727 f.write(" if TYPE_CHECKING:"+"\n");
3728 f.write(" class .*\bProtocol\‍):"+"\n");
3729 f.write(" @(abc\.)?abstractmethod"+"\n");
3730 ## Check whether a configuration file has been given
3731 # If that is the case, copy it into the current temporary working directory, bypassing root
3732 # as both statements are mutually exclusive
3733 if config: shutil.copy(config, os.path.abspath(os.getcwd()))
3734 # Explicit definition of the root directory
3735 else: command.extend(["--rootdir=%s" % os.path.join(os.getcwd())])
3736 # Modify default settings path
3737 command.extend(["--cov-config=.coveragerc"])
3738 # Change default import mode
3739 command.extend(["--import-mode=importlib"])
3740 # Explicit definition of the test directory. Everything else is excluded.
3741 command.extend(['--override-ini="testpaths=%s"' % str(test_directory)])
3742 # Print all test cases and passes in a detailed manner
3743 if self.verboseverbose >=1: command.extend(['--verbose'])
3744 if self.verboseverbose >=2: command.extend(['-rA'])
3745 # Add the created test folder to the final command
3746 command.extend([os.path.join(os.getcwd(),test_directory)]); pytest.main(command)
3747 # Move all results to the output directory
3748 result = [x for x in os.listdir(os.getcwd()) if x.startswith(("coverage", "htmlcov", "junit",))]
3749 # Copy all files and folders to the output directory
3750 for x in result:
3751 # Remove any preexisting folders. Folders are recreated in the process.
3752 if os.path.isdir(x):
3753 try: shutil.rmtree(os.path.join(self.outdiroutdir,x));
3754 except: pass
3755 finally: shutil.copytree(x,os.path.join(self.outdiroutdir,x));
3756 # Copy data
3757 else: shutil.copy(x, os.path.join(self.outdiroutdir,x))
3758 # Recover the initial command
3759 sys.argv = sys_arg_recovery
3760 # Recover the old system path.
3761 sys.path = sys_path_recovery
3762 pass
3763
3764## @class PyXMake.Build.Make.Sphinx
3765# Base class for all Sphinx build events. Inherited from Make.
3766class Sphinx(Make):
3767 """
3768 Inherited class to automatically build a documentation using Sphinx.
3769 """
3770 def __init__(self, *args,**kwargs):
3771 """
3772 Initialization of sphinx class object.
3773 """
3774 super(Sphinx, self).__init__(*args,**kwargs)
3775 ## String identifier of current instance.
3776 self.MakeObjectKind = 'Sphinx'
3777
3778 # Validate third party dependencies
3779 from six import exec_
3780 from PyXMake.Build import __install__ #@UnresolvedImport
3781 exec_(open(__install__.__file__).read(),{"__name__": "__main__", "__file__":__install__.__file__})
3782
3783 # Build script for Sphinx documentation
3784 os.environ["PATH"] += os.pathsep + os.pathsep.join([os.path.join(os.path.dirname(sys.executable),"Scripts")])
3785
3786 # Immutable settings for Sphinx object
3787 ## Path to Sphinx executable.
3788 self.path2exe = os.path.join(os.path.dirname(sys.executable),"Scripts")
3789 ## Executable of Sphinx.
3790 self.exe = 'sphinx-build.exe'
3791 ## Ordered dictionary containing all available options
3792 self.BuildOption = OrderedDict()
3793
3794 # Project name from BuildID
3795 os.environ["sphinx_project"] = str(self.buildid)
3796
3797 # Name of index file (master document).
3798 os.environ["sphinx_master"] = str(self.srcs[0])
3799
3800 # Default color scheme
3801 from PyXMake.Build.config.stm_color import DLRBlue as sphinx_color #@UnresolvedImport
3802 os.environ["sphinx_color"] = sphinx_color
3803
3804 # Remove MKL from default command line
3805 ## Blank version of list containing library directories without initially specifying MKL.
3807 self.libdirslibdirs = []
3808 pass
3809
3810 def Settings(self, **kwargs): # pragma: no cover
3811 """
3812 Define environment variables for the default configuration file.
3813 """
3814 delimn = "_"
3815 # Set environment variables for configuration script
3816 for key, value in kwargs.items():
3817 os.environ[delimn.join(["sphinx",str(key)])] = str(value)
3818 pass
3819
3820 @classmethod
3821 def parse(cls, **kwargs): # pragma: no cover
3822 """
3823 Execute the current class as a CLI command.
3824 """
3825 # Import its main from VTL
3826 from PyXMake.VTL import sphinx
3827 # Evaluate current command line
3828 command = kwargs.get("command",sys.argv)
3829 # Process all known arguments
3830 parser = argparse.ArgumentParser(description='CLI wrapper options for Sphinx with more sensible default settings.')
3831 parser.add_argument('name', type=str, nargs=1, help="Name of the project")
3832 parser.add_argument('source', type=str, nargs=1, help="Absolute path to the source file directory.")
3833 parser.add_argument('-f', '--file', nargs='+', default=[], help="Top level input file used in the creation of the documentation.")
3834 parser.add_argument('-l', '--logo', nargs='+', default=[], help="Top level logo in SVG format.")
3835 parser.add_argument('-i', '--include', nargs='+', default=[], help="Additional files defining test cases required for test coverage.")
3836 parser.add_argument('-s', '--scratch', nargs='+', default=[], help="Default scratch folder for Sphinx. Defaults to current workspace.")
3837 parser.add_argument("-o","--output", type=str, nargs=1, help="Absolute path to output directory. Defaults to current project folder.")
3838 parser.add_argument('-t', '--theme', type=str, nargs=1, help="An installed Sphinx theme. Defaults to 'Read the docs' theme.")
3839 parser.add_argument("-v","--verbosity", type=str, nargs=1, help="Level of verbosity. Defaults to 0 - meaning no output. Max value is 2.")
3840 parser.add_argument('--icon', type=str, nargs=1, help=argparse.SUPPRESS)
3841 # Check all options or run unit tests in default mode
3842 try:
3843 # Check CLI options
3844 _ = command[1]
3845 args, _ = parser.parse_known_args(command[1:])
3846 # Project name is mandatory
3847 project = Utility.GetSanitizedDataFromCommand(args.name, is_path=False)[0]
3848 ## Specification of main input file.
3849 mainfile = args.file[0];
3850 # Specification of source directory is mandatory
3851 source = os.path.abspath(args.source[0]) ;
3852 # Optional non-default definition of additional tests cases
3853 try:
3854 _ = args.include[0]
3855 # Collect all given paths. Get system independent format
3856 include = Utility.GetSanitizedDataFromCommand(args.include)
3857 # No extra test cases have been given
3858 except: include = []
3859 # Optional non-default output directory
3860 try: output = args.output[0]
3861 except: output = os.path.abspath(os.getcwd())
3862 # Optional non-default scratch directory
3863 try: scratch = args.scratch[0]
3864 except: scratch = os.path.abspath(os.getcwd())
3865 # Optional non-default icon
3866 try: icon = args.logo[0]
3867 except: icon = getattr(args, "icon", [None] )
3868 if icon: icon = os.path.abspath(next(iter(icon)))
3869 # Optional non-default theme
3870 try: theme = args.theme[0]
3871 except: theme = None
3872 # Verbose output level. Defaults to ZERO - meaning no output. Can be increased to 2.
3873 try: verbosity = int(args.verbosity[0])
3874 except: verbosity = 2
3875 # Create a dictionary combining all settings
3876 settings = {"source":source, "output":output, "include":include, "scratch": scratch, "verbosity":verbosity, "logo":icon}
3877 # Parse additional settings to modify the build
3878 if theme: settings.update({"html_theme":theme})
3879
3880 # Use an exception to allow help message to be printed.
3881 except Exception as _:
3882 # Execute default.
3883 sphinx("Composite Damage Analysis Code", "codac")
3884 else:
3885 # Execute CLI command
3886 sphinx(project, mainfile, **settings)
3887 pass
3888
3889 def create(self, **kwargs): # pragma: no cover
3890 """
3891 Execute make command
3892 """
3893 from distutils.dir_util import copy_tree
3894
3895 # You can set these variables from the command line.
3896 self.SPHINXOPTS = os.getenv('sphinx_opts', '')
3897 self.SPHINXBUILD = os.getenv('sphinx_build', 'sphinx-build')
3898 self.PAPER = os.getenv('sphinx_paper', None)
3899
3900 self.SOURCEDIR = os.getenv('sphinx_sourcedir', self.srcdir)
3901 self.BUILDDIR = os.getenv('sphinx_builddir', Utility.PathLeaf(tempfile.NamedTemporaryFile().name))
3902 self.TEMPDIR = os.getenv('sphinx_templates', "_templates")
3903 self.STATICEDIR = os.getenv('sphinx_static', "_static")
3904
3905 # Create static and template folder relative to source directory, if required.
3906 os.environ['sphinx_static'] = self.STATICEDIR; os.environ['sphinx_templates'] = self.TEMPDIR
3907
3908 # Add path dependencies
3909 os.environ["sphinx_include"] = os.getenv("sphinx_include","")
3910 os.environ["sphinx_include"] += os.pathsep + os.pathsep.join(self.incdirsincdirs)
3911
3912 def validate():
3913 """
3914 User-friendly check for sphinx-build
3915 """
3916 with open(os.devnull, 'w') as devnull:
3917 try:
3918 if subprocess.call([self.SPHINXBUILD, '--version'],stdout=devnull, stderr=devnull) == 0:
3919 return
3920 except FileNotFoundError:
3921 pass
3922 print(
3923 "The '{0}' command was not found. Make sure you have Sphinx "
3924 "installed, then set the SPHINXBUILD environment variable "
3925 "to point to the full path of the '{0}' executable. "
3926 "Alternatively you can add the directory with the "
3927 "executable to your PATH. If you don't have Sphinx "
3928 "installed, grab it from http://sphinx-doc.org/)"
3929 .format(self.SPHINXBUILD))
3930 sys.exit(1)
3931 return
3932
3933 def build(builder, success_msg=None, extra_opts=None, outdir=None,doctrees=True):
3934 """
3935 The default target
3936 """
3937 builddir = os.path.join(self.BUILDDIR or outdir)
3938 command = [self.SPHINXBUILD, '-M', builder, self.SOURCEDIR, builddir]
3939 command.extend(['-c', os.getenv('sphinx_config',self.SOURCEDIR)])
3940 if command[-1] == self.SOURCEDIR:
3941 command = command[:len(command)-2]
3942 if doctrees:
3943 command.extend(['-d', os.path.join(self.BUILDDIR, 'doctrees')])
3944 if extra_opts:
3945 command.extend(extra_opts)
3946 command.extend(shlex.split(self.SPHINXOPTS))
3947 # Execute build command
3948 if Utility.Popen(command,self.verboseverbose).returncode == 0:
3949 print('Build finished. ' + success_msg.format(self.outdiroutdir or builddir))
3950
3951 def buildmethod(function):
3952 """
3953 Decorator function for each build option
3954 """
3955 self.BuildOption[function.__name__] = function
3956 return function
3957
3958 @buildmethod
3959 def default():
3960 """
3961 The default target
3962 """
3963 return html()
3964
3965 @buildmethod
3966 def clean():
3967 """
3968 Remove the build directory
3969 """
3970 shutil.rmtree(self.BUILDDIR, ignore_errors=True)
3971
3972 @buildmethod
3973 def html():
3974 """
3975 Make standalone HTML files
3976 """
3977 return build('html', 'The HTML pages are in {}.')
3978
3979 @buildmethod
3980 def dirhtml():
3981 """
3982 Make HTML files named index.html in directories
3983 """
3984 return build('dirhtml', 'The HTML pages are in {}')
3985
3986 @buildmethod
3987 def singlehtml():
3988 """
3989 Make a single large HTML file
3990 """
3991 return build('singlehtml', 'The HTML page is in {}.')
3992
3993 @buildmethod
3994 def pickle():
3995 """
3996 Make pickle files
3997 """
3998 return build('pickle', 'Now you can process the pickle files.')
3999
4000 @buildmethod
4001 def json():
4002 """
4003 Make JSON files
4004 """
4005 return build('json', 'Now you can process the JSON files.')
4006
4007 @buildmethod
4008 def htmlhelp():
4009 """
4010 Make HTML files and a HTML help project
4011 """
4012 return build('htmlhelp', 'Now you can run HTML Help Workshop with the .hhp project file in {}.')
4013
4014 @buildmethod
4015 def qthelp():
4016 """
4017 Make HTML files and a qthelp project
4018 """
4019 return build('qthelp', 'Now you can run "qcollectiongenerator" with the '
4020 '.qhcp project file in {0}, like this: \n'
4021 '# qcollectiongenerator {0}/RinohType.qhcp\n'
4022 'To view the help file:\n'
4023 '# assistant -collectionFile {0}/RinohType.qhc')
4024
4025 @buildmethod
4026 def devhelp():
4027 """
4028 Make HTML files and a Devhelp project
4029 """
4030 return build('devhelp', 'To view the help file:\n'
4031 '# mkdir -p $HOME/.local/share/devhelp/RinohType\n'
4032 '# ln -s {} $HOME/.local/share/devhelp/RinohType\n'
4033 '# devhelp')
4034
4035 @buildmethod
4036 def epub(self):
4037 """
4038 Make an epub
4039 """
4040 return self.build('epub', 'The epub file is in {}.')
4041
4042 @buildmethod
4043 def rinoh(self):
4044 """
4045 Make a PDF using rinohtype
4046 """
4047 return self.build('rinoh', 'The PDF file is in {}.')
4048
4049 @buildmethod
4050 def latex():
4051 """
4052 Make LaTeX files, you can set PAPER=a4 or PAPER=letter
4053 """
4054 extra_opts = ['-D', 'latex_paper_size={}'.format(self.PAPER)] if self.PAPER else None
4055 return build('latex', 'The LaTeX files are in {}.\n'
4056 "Run 'make' in that directory to run these through "
4057 "(pdf)latex (use the 'latexpdf' target to do that "
4058 "automatically).", extra_opts)
4059
4060 @buildmethod
4061 def latexpdf():
4062 """
4063 Make LaTeX files and run them through pdflatex
4064 """
4065 _ = latex()
4066 print('Running LaTeX files through pdflatex...')
4067 builddir = os.path.join(self.BUILDDIR, 'latex')
4068 subprocess.call(['make', '-C', builddir, 'all-pdf'])
4069 print('pdflatex finished; the PDF files are in {}.'.format(builddir))
4070
4071 @buildmethod
4072 def latexpdfja():
4073 """
4074 Make LaTeX files and run them through platex/dvipdfmx
4075 """
4076 _ = latex()
4077 print('Running LaTeX files through platex and dvipdfmx...')
4078 builddir = os.path.join(self.BUILDDIR, 'latex')
4079 subprocess.call(['make', '-C', builddir, 'all-pdf-ja'])
4080 print('pdflatex finished; the PDF files are in {}.'.format(builddir))
4081
4082 @buildmethod
4083 def text():
4084 """
4085 Make text files
4086 """
4087 return build('text', 'The text files are in {}.')
4088
4089 @buildmethod
4090 def man():
4091 """
4092 Make manual pages
4093 """
4094 return build('man', 'The manual pages are in {}.')
4095
4096 @buildmethod
4097 def texinfo():
4098 """
4099 Make TexInfo files
4100 """
4101 return build('texinfo', 'The Texinfo files are in {}.\n'
4102 "Run 'make' in that directory to run these "
4103 "through makeinfo (use the 'info' target to do "
4104 "that automatically).")
4105
4106 @buildmethod
4107 def info():
4108 """
4109 Make Texinfo files and run them through makeinfo
4110 """
4111 _ = texinfo()
4112 print('Running Texinfo files through makeinfo...')
4113 builddir = os.path.join(self.BUILDDIR, 'texinfo')
4114 subprocess.call(['make', '-C', builddir, 'info'])
4115 print('makeinfo finished; the Info files are in {}.'.format(builddir))
4116
4117 @buildmethod
4118 def gettext():
4119 """
4120 Make PO message catalogs
4121 """
4122 return build('gettext', 'The message catalogs are in {}.', outdir='locale',doctrees=False)
4123
4124 @buildmethod
4125 def changes():
4126 """
4127 Make an overview of all changed/added/deprecated items
4128 """
4129 return build('changes', 'The overview file is in {}.')
4130
4131 @buildmethod
4132 def xml():
4133 """
4134 Make Docutils-native XML files
4135 """
4136 return build('xml', 'The XML files are in {}.')
4137
4138 @buildmethod
4139 def pseudoxml():
4140 """
4141 Make pseudoxml-XML files for display purposes
4142 """
4143 return self.build('pseudoxml', 'The pseudo-XML files are in {}.')
4144
4145 @buildmethod
4146 def linkcheck():
4147 """
4148 Check all external links for integrity
4149 """
4150 return build('linkcheck', 'Look for any errors in the above output or in {}/output.txt.')
4151
4152 @buildmethod
4153 def doctest():
4154 """
4155 Run all doctests embedded in the documentation (if enabled)
4156 """
4157 return build('doctest', 'Look at the results in {}/output.txt.')
4158
4159 @buildmethod
4160 def assist():
4161 """
4162 List all targets
4163 """
4164 print("Please use '{} <target>' where <target> is one of" .format(sys.argv[0]))
4165 width = max(len(name) for name in self.BuildOption)
4166 for name, target in self.BuildOption.items():
4167 print(' {name:{width}} {descr}'.format(name=name, width=width, descr=target.__doc__))
4168
4169 # Validate installation status
4170 validate()
4171 # Get additional command line arguments
4172 args = ['default'] or sys.argv[1:]
4173 for arg in args:
4174 # Create a temporary build directory. We do not need unsuccessful build
4175 with Utility.TemporaryDirectory(self.scrtdirscrtdir):
4176 # Create a new local directory
4177 temp = Utility.PathLeaf(tempfile.NamedTemporaryFile().name)
4178 # Copy all source files into the temporary directory
4179 shutil.copytree(os.path.abspath(self.SOURCEDIR), os.path.abspath(temp), ignore=shutil.ignore_patterns(".git",".svn")); self.SOURCEDIR = temp
4180 # Add auto documentation feature
4181 for x in os.getenv("sphinx_include","").split(os.pathsep):
4182 try: subprocess.call(["sphinx-apidoc","-o",os.path.join(temp,"_advanced",Utility.PathLeaf(os.path.dirname(x)).lower()),x])
4183 except: pass
4184 # If no configuration file is found in the source folder, use the default template
4185 if not os.path.exists(os.path.join(temp,"conf.py")):
4186 # Rename default configuration file to match naming convention.
4187 copyfile(os.path.join(PyXMakePath,"Build","config","stm_conf.py"), os.path.join(temp,"conf.py"))
4188 # Create an additional static folder if not already existing
4189 if not os.path.exists(os.path.join(temp,"_static")):
4190 os.mkdir(os.path.join(temp,'_static')); copyfile(os.path.join(PyXMakePath,"Build","config","stm_style.css"), os.path.join(temp,"_static","style.css"))
4191 # Create an additional templates folder
4192 if not os.path.exists(os.path.join(temp,"_templates")):
4193 os.mkdir(os.path.join(temp,'_templates')); copyfile(os.path.join(PyXMakePath,"Build","config","stm_layout.html"), os.path.join(temp,"_templates","layout.html"))
4194 # Create all documentations
4195 self.BuildOption[arg]()
4196 # Do not keep temporary tree by default
4197 if not kwargs.get("keep_doctree",False):
4198 try:
4199 shutil.rmtree(os.path.join(self.BUILDDIR,"doctrees"))
4200 except OSError:
4201 pass
4202 # Copy results to output directory
4203 copy_tree(self.BUILDDIR, self.outdiroutdir)
4204
4205 # Return success
4206 return 0
4207
4208## @class PyXMake.Build.Make.SSH
4209# Base class for all build events requiring a SSH connection. Inherited from Make.
4210class SSH(Make): # pragma: no cover
4211 """
4212 Inherited class for all builds using SSH connection.
4213 """
4214 def __init__(self, *args, **kwargs):
4215 """
4216 Initialization of SSH class object.
4217 """
4218 super(SSH, self).__init__(*args, **kwargs)
4219
4220 # Add Fortran wrapper function to SSH class.
4221 setattr(self, str(Fortran.Wrapper.__name__), MethodType(Fortran.Wrapper, self))
4222
4223 # Defined here to be checked later.
4224 ## Wrapper interface file for 3rd party FORTRAN code. Automatically creates a module of the underlying source material.
4226 self.wrapper_source = ""
4227 self.wrapper_module = "pyx_module.f90"
4228
4229 ## String identifier of current instance.
4230 self.MakeObjectKind = 'SSH'
4231
4232 # Immutable settings for SSH object
4233 ## Name of library, assembled using BuildID.
4234 self.libname = "lib"+self.buildid + self.architecture
4235 ## Temporary build name.
4236 self.buildnamebuildname = self.buildid+'_ssh'
4237 ## Environment variables to be set prior to the execution of the build command. Intel Fortran 12+
4238 self.export = "export CPATH=$CPATH"
4239 ## Environment variables to be set prior to the execution of the build command. Intel Fortran 11 and lower.
4240 self.__old_export = "export FPATH=$FPATH"
4241 ## Environment variable to be set prior to the execution of the build command.
4242 self.__library_path = "export LIBRARY_PATH=$LIBRARY_PATH"
4243 ## Custom intel path
4244 self.__intel_path = "export pyx_ifort=$(which ifort)"
4245
4246 ## Define if the input should be compiled exactly as provided.
4247 # Defaults to False, meaning that merging & pre-processing utilities will be carried out.
4248 self.incrementalincremental = kwargs.get('incremental', False)
4249
4250 # Initialization of lists containing additional sources, modules or libraries
4251 ## List of libraries which should be statically linked in.
4252 self.linkedIn = []
4253
4254 # Initialization of tuple containing temporary files
4255 ## Blank version of tuple to store temporary file names scheduled for removal.
4256 self.tempstemps = ()
4257
4258 ## Load an additional library prior to execution of all commands. Defaults to an empty string.
4259 self.environment = ""
4260
4261 # Remove MKL from default command line
4262 ## Blank version of list containing library directories. MKL library has been removed since location
4263 # on the SSH remote computer is not known a priori.
4265 pass
4266
4267 def OutputPath(self, libpath, modulepath=None):
4268 """
4269 Define output directories for modules and libraries. Output written to the workspace is DELETED.
4270 """
4271 # Output module files to scratch directory by default.
4272 if modulepath is None:
4273 modulepath = libpath
4274 ## Output path for module or header files.
4275 self.outmodule = modulepath + posixpath.sep
4276 ## Output path for library files.
4277 self.outlibs = libpath + posixpath.sep
4278 pass
4279
4280 def Settings(self, user, key="", host='129.247.54.37', port=22, use_cache=True, **kwargs):
4281 """
4282 Define settings to establish a SSH connection.
4283 """
4284 # Establish SSH connection to institute cluster // mg 07.08.17
4285 # Parse an external client directly to the underlying connection. Use this connection for all following operations.
4286 if kwargs.get("client",None):
4287 self.ssh_client = kwargs.get("client");
4288 sftp = self.ssh_client.open_sftp(); sftp.chdir(".")
4289 self.workspace = kwargs.get("workspace",posixpath.join(sftp.getcwd(),"")); sftp.close()
4290 return
4291 ## Remote workspace. This is the scratch directory for the build event.
4292 # Defaults to /home/user/.
4293 self.workspace = kwargs.get("workspace",posixpath.join(Utility.AsDrive('home',posixpath.sep),user)+posixpath.sep)
4294 ## Instance of SSHClient to establish a SSH connection.
4295 self.ssh_client = paramiko.SSHClient()
4296 self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
4297 # Read password (if given)
4298 password = kwargs.pop("password","");
4299 try:
4300 # A password has been given. Use it directly
4301 if password: raise ValueError
4302 # Try to connect using key file
4303 try: self.ssh_client.connect(hostname=host, port=port, username=user, key_filename=key, timeout=kwargs.get("timeout",None))
4304 except paramiko.ssh_exception.SSHException:
4305 self.ssh_client.connect(hostname=host, port=port, username=user, key_filename=key, timeout=kwargs.get("timeout",None),
4306 disabled_algorithms=kwargs.get("disabled",{"pubkeys":["rsa-sha2-512", "rsa-sha2-256"]}))
4307 except socket.timeout: raise TimeoutError
4308 except:
4309 if use_cache:
4310 # Check if the password should be cached. Active by default.
4311 try: import keyring
4312 except: raise ModuleNotFoundError
4313 # Check if the stored/cached password can be used. Prompt to reenter the password if that is not the case.
4314 try: self.ssh_client.connect(hostname=host, port=port, username=user, password=keyring.get_password(user, user))
4315 except:
4316 # Get password from user
4317 if not password: password = getpass.getpass()
4318 keyring.set_password(user, user, password)
4319 finally: self.ssh_client.connect(hostname=host, port=port, username=user, password=keyring.get_password(user, user))
4320 else:
4321 if not password: password = getpass.getpass()
4322 self.ssh_client.connect(hostname=host, port=port, username=user, password=password)
4323 pass
4324
4325 def Environment(self, path="", bash="", args="", method="source"):
4326 """
4327 Load an additional environment file prior to execution of all commands.
4328 """
4329 source = method
4330 ## Execute an additional bash script prior to all build commands.
4331 if any([str(posixpath.join(path,bash)).startswith("module"),args.startswith("module")]): source = ""
4332 self.environment += " ".join([source,posixpath.join(path,bash),args])+"; "
4333 pass
4334
4335 def Postprocessing(self, cmdstring=""):
4336 """
4337 Assemble command string for the post-build event.
4338 """
4339 ## Command executed during post-build event.
4340 self.postcmdpostcmd = cmdstring
4341 pass
4342
4343 def Build(self, cmdstring, run= "ifort", path="", lib= "", linkedIn="", **kwargs):
4344 """
4345 Assemble command strings for the main build event.
4346 """
4347 cmd = ""; self.postcmdpostcmd = ""
4348
4349 # Which libraries are used for linking (relative path).
4350 self.libslibs.append(lib)
4351 self.libslibs = list(Utility.ArbitraryFlattening(self.libslibs))
4352
4353 # Which libraries have to be additionally linked in (absolute path).
4354 self.linkedIn.append(linkedIn)
4355 self.linkedIn = list(Utility.ArbitraryFlattening(self.linkedIn))
4356
4357 # Build commands using Intel Fortran (mutable)
4358 ## (Intel Fortran) Compiler Path
4359 self.path2exe = path; self.exe = run
4360
4361 # Check whether an interface module wrapper was added to the current folder
4362 if os.path.isfile(self.intermediate_wrapper):
4363 if (Utility.IsNotEmpty(self.wrapper_source)):
4365
4366 # Immutable settings for SSH object
4367 if self.incrementalincremental:
4368 c_files = [x for x in self.srcssrcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]
4369 cmd += ' %s ' % (' '.join(c_files))
4370
4371 # Always add MKL when used with f2py.
4372 if self.exe in ("f2py") and self.exe:
4373 cmd += "-m "+str(self.buildid+self.architecture)
4374 # Assist f2py by providing shared MKL libraries directly
4375 self.libslibs.extend(["mkl_rt","iomp5", "pthread", "m", "dl"])
4376
4377 # Add libraries for referencing to the command string (they are only used as resources)
4378 for x in [' -l'+x+' ' for x in self.libslibs if x]:
4379 cmd += x
4380
4381 ## Remote (Intel) Compiler command.
4382 if self.exe and self.exe not in ("custom"):
4383 self.makecmdmakecmd = posixpath.join(self.path2exe,self.exe)+" -c "+ cmd + cmdstring
4384 else:
4385 self.makecmdmakecmd = cmdstring
4386
4387 ## Remote Linker command.
4388 if self.exe not in ("ifort", "gcc", "g++"):
4389 ## Do no use any archiver
4390 self.linkcmd = ""
4391 elif self.exe == "ifort":
4392 ## Remote Intel Linker command.
4393 self.linkcmd = posixpath.join(self.path2exe,"xiar")+" -rc "
4394 else:
4395 ## Simply execute UNIX archiver
4396 self.linkcmd = posixpath.join("","ar")+" -rc "
4397 pass
4398
4399 def create(self, **kwargs):
4400 """
4401 Define settings to establish SSH connection.
4402 """
4403 cmd = ""
4404
4405 # Add all include paths to CPATH & FPATH (deprecated!) environment variable
4406 includes = [':"'+x+'"' for x in self.incdirs]
4407 for x in includes:
4408 self.export += x
4409 self.__old_export += x
4410 self.export += " && " + self.__old_export
4411
4412 # Add all library paths to LIBRARY_PATH environment variable
4413 library = [':"'+x+'" ' for x in self.libdirslibdirs]
4414 for x in library:
4415 self.__library_path += x
4416 self.export += " && " + self.__library_path
4417
4418 # Add libraries for linking to the command string
4419 try:
4420 if self.linkedIn[0] != "":
4421 linklist = ['ar -x "'+x+'" && ' for x in self.linkedIn]
4422 for x in linklist:
4423 cmd += x
4424 # Link list is empty or does not exist
4425 except IndexError:
4426 pass
4427
4428 # Get the target and the base name of library (created in the process).
4429 target = posixpath.join(self.workspace,self.buildnamebuildname)
4430 # base = os.path.splitext(target)[0]
4431
4432 # Go into scratch directory (if defined)
4434
4435 # Pre-build event (if required)
4436 try:
4437 if self.precmdprecmd != '':
4438 Utility.Popen(self.precmdprecmd, self.verboseverbose)
4439 except:
4440 pass
4441
4442 # Establish ssh connection and execute make commands on the linux cluster.
4443 sftp = self.ssh_client.open_sftp()
4444
4445 try:
4446 sftp.put(self.buildnamebuildname,target)
4447 if os.path.isfile(self.intermediate_wrapper):
4448 # Use general-purpose wrapper file
4449 Utility.ReplaceTextinFile(self.intermediate_wrapper, self.wrapper_module, {'%pyx_source%':'"'+self.buildnamebuildname+'"',"#":" "}, source=self.scrtdirscrtdir)
4450 sftp.put(os.path.join(self.scrtdirscrtdir,self.wrapper_module),posixpath.join(self.workspace, self.wrapper_module))
4451 target = posixpath.join(self.workspace,self.wrapper_module)
4452 # Preserve the unique name of each object file
4453 self.makecmdmakecmd += " -o "+ os.path.splitext(self.buildnamebuildname)[0]+".o"
4454 except:
4455 target = ""
4456 for cs in self.srcssrcs:
4457 sftp.put(os.path.join(self.scrtdirscrtdir,cs),posixpath.join(self.workspace,cs))
4458
4459 # Put f2py mapping file into current workspace
4460 if self.exe not in ("ifort", "gcc", "g++","custom"):
4461 sftp.put(os.path.join(Path2Config,".f2py_f2cmap"), posixpath.join(self.workspace,".f2py_f2cmap"))
4462 sftp.close()
4463
4464 # Create output folder if not existing
4465 Utility.SSHPopen(self.ssh_client,"mkdir -p "+self.outmodule+"; mkdir -p "+self.workspace,self.verboseverbose, **kwargs)
4466
4467 # Delete old content in output folders
4468 if not kwargs.get("combine", False):
4469 Utility.SSHPopen(self.ssh_client,'rm -f '+self.outmodule+'*.mod; rm -f '+self.workspace+'*.mod', self.verboseverbose, **kwargs)
4470 else:
4471 Utility.SSHPopen(self.ssh_client,'rm -f '+self.workspace+'*.mod',self.verboseverbose, **kwargs)
4472
4473 self.command = self.environment + self.export + " && cd "+self.workspace+ " && " + self.makecmdmakecmd
4474 if Utility.IsNotEmpty(target):
4475 self.command+= ' "'+target+'"'
4476 sbuild = Utility.SSHPopen(self.ssh_client, self.command, self.verboseverbose, **kwargs)
4477
4478 if sbuild==0:
4479 # There is a valid link command. Use it
4480 if Utility.IsNotEmpty(self.linkcmd):
4481 self.command = self.environment + self.export + " && cd "+self.workspace+ " && " + cmd + self.linkcmd + self.libname+'.a '+'*.o '
4482 sarch = Utility.SSHPopen(self.ssh_client, self.command, self.verboseverbose, **kwargs)
4483 elif self.exe in ["custom"]:
4484 sarch = 1; scopy = 1
4485 # Copy all created shared libraries.
4486 else:
4487 sarch = 1; scopy = 0
4488 self.postcmdpostcmd += ' && cp -rf '+self.workspace+'*.so '+self.outlibs
4489
4490 if sarch==0:
4491 self.command = 'cp -rf '+self.workspace+'*.mod '+self.outmodule+'; cp -rf '+self.workspace+'*.a '+self.outlibs
4492 scopy = Utility.SSHPopen(self.ssh_client, self.command, self.verboseverbose, **kwargs)
4493
4494 if scopy == 0:
4495 Utility.SSHPopen(self.ssh_client,'rm '+self.workspace+'*.o; rm '+self.workspace+'*.mod; rm '+self.workspace+'*.a; rm '+
4496 self.workspace+'*.f90; rm -rf '+posixpath.join(self.workspace,"intel"), self.verboseverbose, **kwargs)
4497
4498 if Utility.IsNotEmpty(self.postcmdpostcmd):
4499 self.export += " && " + self.__intel_path
4500 self.command = self.environment + self.export + " && " + self.postcmdpostcmd
4501 spost = Utility.SSHPopen(self.ssh_client, self.command, self.verboseverbose, **kwargs)
4502 if spost == 0:
4503 Utility.SSHPopen(self.ssh_client,'rm -f '+self.workspace+'*.o; rm '+self.workspace+'*.mod; rm -f '+self.workspace+'*.a; rm -f '+
4504 self.workspace+'*.f90; rm -f '+self.workspace+'*.f; rm -rf '+
4505 posixpath.join(self.workspace,"intel")+'; rm -f '+
4506 posixpath.join(self.workspace,".f2py_f2cmap") +
4507 " rm -f %s " % (' '.join([posixpath.join(self.workspace,cs) for cs in self.srcssrcs])),
4508 self.verboseverbose, **kwargs)
4509 pass
4510 pass
4511
4512 # Combine event (needed for TOMS). Combine multiple libraries into ONE.
4513 if kwargs.get("combine", False):
4514 librarian = 'ar'; ext = '.a'; decomp = " && "+librarian+" -x "
4515 mergedid = "lib"+posixpath.basename(self.outmodule.rstrip("/"))
4516 _ , stdout, _ = self.ssh_client.exec_command('ls '+self.outlibs)
4517 multi_libs = [x for x in [x.rstrip("\n") for x in stdout.readlines() if x.startswith(mergedid)]]
4518
4519 try:
4520 # Remove old combined library from the list.
4521 multi_libs.remove(mergedid+self.architecture+ext)
4522 except:
4523 pass
4524
4525 self.postcmdpostcmd = self.environment + self.export + " && cd "+self.outlibs.rstrip("/")+" && "
4526 self.postcmdpostcmd += librarian +" -x " + decomp.join(multi_libs) +" && "
4527 self.postcmdpostcmd += librarian + " -rc " + mergedid+self.architecture+ext+ " *.o"
4528
4529 Utility.SSHPopen(self.ssh_client, self.postcmdpostcmd, self.verboseverbose,**kwargs)
4530 for lib in multi_libs:
4531 Utility.SSHPopen(self.ssh_client,'rm -f '+posixpath.join(self.outlibs,lib),self.verboseverbose, **kwargs)
4532 self.ssh_client.exec_command('rm -f '+self.outlibs+'*.o')
4533
4534 # Go into scratch directory (if defined)
4536
4537 # Finish and delete redundant files
4538 Utility.DeleteFilesbyEnding(self.tempstemps)
4539
4540## Backwards compatibility for deprecated class calls
4541setattr(sys.modules[__name__],"Robot", Coverage)
4542## Forward compatibility for future class calls
4543setattr(sys.modules[__name__],"CMake", Custom)
4544
4545if __name__ == '__main__':
4546 pass
Base class for all C/C++ build events inherited from Make.
incremental
Define if the input should be compiled exactly as provided.
Definition Make.py:1348
__init__(self, *args, **kwargs)
Definition Make.py:1332
str MakeObjectKind
String identifier of current instance.
Definition Make.py:1338
libname
Name of library, assembled using BuildID.
Definition Make.py:1357
bool isStatic
Static or dynamic link library flag.
Definition Make.py:1344
OutputPath(self, libpath=os.getcwd())
Definition Make.py:1381
str exe
The executable command used in the main build event.
Definition Make.py:1341
compargs
Used defined command line options.
Definition Make.py:1411
Build(self, cmdstring, **kwargs)
Definition Make.py:1389
# pragma no cover isStatic
Definition Make.py:1402
list linkedIn
List of libraries which should be statically linked in.
Definition Make.py:1363
buildname
Temporary build name.
Definition Make.py:1359
parse(cls, **kwargs)
Definition Make.py:1423
str linkcmd
Intel Linker command.
Definition Make.py:1416
outlibs
Output path for library files.
Definition Make.py:1386
str makecmd
Intel Compiler command.
Definition Make.py:1413
tuple temps
Blank version of tuple to store temporary file names scheduled for removal.
Definition Make.py:1367
list libdirs
Blank version of list containing library directories without initially specifying MKL.
Definition Make.py:1371
Base class for all Coverage build events.
str MakeObjectKind
String identifier of current instance.
Definition Make.py:3493
__init__(self, *args, **kwargs)
Definition Make.py:3481
create(self, **kwargs)
Definition Make.py:3630
add(cls, *args, **kwargs)
Definition Make.py:3546
parse(cls, **kwargs)
Definition Make.py:3559
Build(self, command=["--cov-fail-under=10","--cov-report=term-missing","--cov-report=html","--cov-report=xml","--junit-xml=junit.xml", "-W ignore::pytest.PytestAssertRewriteWarning"], **kwargs)
Definition Make.py:3620
Base class for all custom build events inherited from Make.
buildname
Immutable settings for Custom object.
Definition Make.py:1132
parse(cls, **kwargs)
Definition Make.py:1267
str MakeObjectKind
String identifier of current instance.
Definition Make.py:1115
str exe
The executable command used in all build events.
Definition Make.py:1118
str makecmd
Command line arguments passed in by the user.
Definition Make.py:1224
temps
Command line arguments passed in by the user.
Definition Make.py:1145
__init__(self, *args, **kwargs)
Definition Make.py:1109
# pragma no cover hasFoss
Definition Make.py:1127
str compargs
Command line arguments passed in by the user.
Definition Make.py:1142
Base class for all Doxygen build events.
parse(cls, **kwargs)
Definition Make.py:2928
path2exe
Path to Doxygen executable.
Definition Make.py:2785
list incdirs
Blank version of list containing library directories without initially specifying MKL.
Definition Make.py:2886
outdir
Output directory of current job.
Definition Make.py:2897
str stype
Type of source file.
Definition Make.py:2806
__init__(self, *args, **kwargs)
Definition Make.py:2770
temps
Tuple of temporary files scheduled for removal.
Definition Make.py:2811
str exe
Executable of Doxygen.
Definition Make.py:2787
Settings(self, brief, header, outdir='', **kwargs)
Definition Make.py:2890
str buildname
Temporary build name of current job.
Definition Make.py:2809
str MakeObjectKind
String identifier of current instance.
Definition Make.py:2776
Base class for all Fortran build events.
str makecmd
Intel Compiler command.
Definition Make.py:1711
compargs
Used defined command line options.
Definition Make.py:1709
OutputPath(self, modulepath=None, libpath=os.getcwd())
Definition Make.py:1602
list libdirs
Blank version of list containing library directories without initially specifying MKL.
Definition Make.py:1595
Build(self, cmdstring, **kwargs)
Definition Make.py:1681
list linkedIn
List of libraries which should be statically linked in.
Definition Make.py:1591
Preprocessing(self, inend='', outend='', copyfiles=[], replace={'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'}, decorator="!DEC$ ATTRIBUTES DLLEXPORT::")
Definition Make.py:1617
Wrapper(self, module_name, source_name=None)
Definition Make.py:1658
create(self, **kwargs)
Definition Make.py:1736
__init__(self, *args, **kwargs)
Definition Make.py:1551
tuple buildname
Temporary build name.
Definition Make.py:1578
tuple linkcmd
Intel Linker command.
Definition Make.py:1727
bool isStatic
Static or dynamic link library flag.
Definition Make.py:1560
str intermediate_wrapper
Wrapper interface file for 3rd party FORTRAN code.
Definition Make.py:1585
outlibs
Output path for library files.
Definition Make.py:1612
str MakeObjectKind
String identifier of current instance.
Definition Make.py:1557
outmodule
Output path for module or header files.
Definition Make.py:1610
tuple libname
Name of library, assembled using BuildID.
Definition Make.py:1576
Base class for all Latex build events.
str MakeObjectKind
String identifier of current instance.
Definition Make.py:3080
parse(cls, **kwargs)
Definition Make.py:3313
show(cls, ProjectID, *args, **kwargs)
Definition Make.py:3183
upload(cls, archive, *args, **kwargs)
Definition Make.py:3227
delete(cls, ProjectID, *args, **kwargs)
Definition Make.py:3293
srcdir
Set default source and output directories for Latex build objects.
Definition Make.py:3110
__init__(self, *args, **kwargs)
Definition Make.py:3071
str path2exe
Path to Latex executable.
Definition Make.py:3103
list incdirs
Blank version of list containing library directories without initially specifying MKL.
Definition Make.py:3118
session(cls, *args, **kwargs)
Definition Make.py:3145
rename(cls, ProjectID, ProjectName, *args, **kwargs)
Definition Make.py:3206
create(self, API="TeXworks", GUI=True, **kwargs)
Definition Make.py:3382
Settings(cls, **kwargs)
Definition Make.py:3136
str exe
Executable of Latex.
Definition Make.py:3105
download(cls, ProjectID, *args, **kwargs)
Definition Make.py:3260
Abstract base class for all make objects.
UseLibraries(self, libs)
Definition Make.py:750
Postprocessing(self, cmdstring='')
Definition Make.py:914
scrtdir
Current scratch directory
Definition Make.py:490
_, self.intelpath, self.iniCompiler intelpath
Path to Intel Fortran Compiler (read from Paths.log or empty).
Definition Make.py:504
compargs
Command line arguments passed in by the user.
Definition Make.py:909
list copyfiles
List of files to be copied to the output directory after finish.
Definition Make.py:520
SourcePath(self, path)
Definition Make.py:758
OutputPath(self, path, files="")
Definition Make.py:769
__posix__(self, **kwargs)
Definition Make.py:602
verbose
Level of verbosity of the current build object.
Definition Make.py:481
AddDependencyPath(self, dependencies)
Definition Make.py:742
bool setarch
Define the architecture for the build directly by using the keyword argument "arch".
Definition Make.py:528
buildid
Base string of build object.
Definition Make.py:468
list srcs
Source file or folders.
Definition Make.py:470
outdir
Default search directory for output.
Definition Make.py:494
str precmd
Command executed during pre-build event.
Definition Make.py:877
__init__(self, BuildID, Srcs, scratch=os.getcwd(), verbose=0, *args, **kwargs)
Definition Make.py:461
msvsc
Default version of Microsoft visual studio used by the Intel Fortran Compiler.
Definition Make.py:531
str architecture
Processor architecture.
Definition Make.py:542
str postcmd
Post build command.
Definition Make.py:572
Preprocessing(self, cmdstring='', inend='', outend='', copyfiles=[], replace={'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'})
Definition Make.py:814
hasFoss
Toggle between free open source software and commercial 3rd party libraries.
Definition Make.py:485
Environment(self, path, script="ifortvars.bat")
Definition Make.py:783
Build(self, cmdstring, **kwargs)
Definition Make.py:883
create(self, **kwargs)
Definition Make.py:1024
F2CPreprocessing(PreprocessingFile)
Definition Make.py:662
list incdirs
List of include directories.
Definition Make.py:512
tuple temps
Tuple of data to be removed after job completion.
Definition Make.py:508
stype
Source file type.
Definition Make.py:476
list libdirs
List of library directories.
Definition Make.py:514
str iniCompiler
Default initialization of compiler script.
Definition Make.py:524
list libs
List of actual libraries (by name) used during linking.
Definition Make.py:516
str makecmd
Command executed during build event.
Definition Make.py:911
sanitize(string, **kwargs)
Definition Make.py:923
setup(*args, **kwargs)
Definition Make.py:937
AddIncludePath(self, includes)
Definition Make.py:734
srcdir
Default search directory for source files.
Definition Make.py:492
run(cls, **kwargs)
Definition Make.py:1015
Base class for all NSIS build events.
create(self, **kwargs)
Definition Make.py:2701
srcdir
Adding option to deactivate NSIS and instead use a compressed archive directly.
Definition Make.py:2722
FTP(self, user, key, upload_file, host='ftp.dlr.de', path="/public/download/nightly")
Definition Make.py:2620
parse(cls, **kwargs)
Definition Make.py:2634
buildid
Adding option to deactivate NSIS and instead use a compressed archive directly.
Definition Make.py:2710
__init__(self, *args, **kwargs)
Definition Make.py:2590
path2exe
Path to NSIS executable.
Definition Make.py:2607
str exe
Executable of NSIS.
Definition Make.py:2609
makecmd
Adding option to deactivate NSIS and instead use a compressed archive directly.
Definition Make.py:2613
outdir
Adding option to deactivate NSIS and instead use a compressed archive directly.
Definition Make.py:2711
verbose
Adding option to deactivate NSIS and instead use a compressed archive directly.
Definition Make.py:2741
str MakeObjectKind
String identifier of current instance.
Definition Make.py:2598
ftp_client
Remote workspace.
Definition Make.py:2627
Abstract base class for all NT subclasses.
str SystemObjectKind
String identifier of current instance.
Definition Make.py:128
__init__(self, *args, **kwargs)
Definition Make.py:122
Abstract base class for all system subclasses.
str SystemObjectKind
String identifier of current instance.
Definition Make.py:110
__init__(self, *args, **kwargs)
Definition Make.py:108
Abstract base class for all POSIX subclasses.
outdir
Accept both OutLibs and OutDir variable.
Definition Make.py:429
__create__(self, **kwargs)
Definition Make.py:154
scrtdir
Overwrite create method in all subclasses to use a predefined MakeFile for all builds.
Definition Make.py:150
__init__(self, *args, **kwargs)
Definition Make.py:139
str SystemObjectKind
String identifier of current instance.
Definition Make.py:145
temps
Overwrite create method in all subclasses to use a predefined MakeFile for all builds.
Definition Make.py:152
Base class for all Py2X (for now only f2py) build events.
str exe
The executable command used in the main build event.
Definition Make.py:1989
no_static_mkl
Define whether Intel's MKL should be statically or dynamically linked.
Definition Make.py:1971
show(package, feature, **kwargs)
Definition Make.py:2120
str path2exe
Absolute system path to Python executable.
Definition Make.py:1987
no_append_arch
Define whether the architecture shall be appended to the build name.
Definition Make.py:1979
__init__(self, *args, **kwargs)
Definition Make.py:1958
inspect(package, **kwargs)
Definition Make.py:2108
str MakeObjectKind
String identifier of current instance.
Definition Make.py:1967
no_mkl
Define whether Intel's MKL should be discarded Defaults to False on NT systems.
Definition Make.py:1975
incremental
Define if the input should be compiled exactly as provided.
Definition Make.py:1983
copyfile(os.path.join(Path2Config,), os.path.join(self.scrtdir,)) bare
Iterate through all active processes matching the current BuildID and kill them.
Definition Make.py:2068
str buildname
Temporary build name of current job.
Definition Make.py:2023
parse(cls, *args, **kwargs)
Definition Make.py:2160
temps
Tuple of temporary files deleted after job completion.
Definition Make.py:2076
Base class for all PyInstaller build events.
Build(self, mode="onefile", **kwargs)
Definition Make.py:2317
__init__(self, *args, **kwargs)
Definition Make.py:2274
create(self, **kwargs)
Definition Make.py:2551
Encryption(self, encrypt, **kwargs)
Definition Make.py:2296
str MakeObjectKind
String identifier of current instance.
Definition Make.py:2282
Preprocessing(self, cmdstring='')
Definition Make.py:2307
precmd
Command executed during pre-build event.
Definition Make.py:2314
Base class for all PyReq build events.
parse(cls, **kwargs)
Definition Make.py:1850
str precmd
Command executed during pre-build event.
Definition Make.py:1834
__init__(self, *args, **kwargs)
Definition Make.py:1818
create(self, **kwargs)
Definition Make.py:1922
Preprocessing(self, cmdstring='')
Definition Make.py:1839
str MakeObjectKind
String identifier of current instance.
Definition Make.py:1826
Base class for all build events requiring a SSH connection.
list libdirs
Blank version of list containing library directories.
Definition Make.py:4264
str MakeObjectKind
String identifier of current instance.
Definition Make.py:4230
str environment
Load an additional library prior to execution of all commands.
Definition Make.py:4259
str intermediate_wrapper
Wrapper interface file for 3rd party FORTRAN code.
Definition Make.py:4225
tuple temps
Blank version of tuple to store temporary file names scheduled for removal.
Definition Make.py:4256
incremental
Define if the input should be compiled exactly as provided.
Definition Make.py:4248
Settings(self, user, key="", host='129.247.54.37', port=22, use_cache=True, **kwargs)
Definition Make.py:4280
outmodule
Output path for module or header files.
Definition Make.py:4275
create(self, **kwargs)
Definition Make.py:4399
__init__(self, *args, **kwargs)
Definition Make.py:4214
workspace
Remote workspace.
Definition Make.py:4289
Postprocessing(self, cmdstring="")
Definition Make.py:4335
list linkedIn
List of libraries which should be statically linked in.
Definition Make.py:4252
outlibs
Output path for library files.
Definition Make.py:4277
str makecmd
Remote (Intel) Compiler command.
Definition Make.py:4383
str buildname
Temporary build name.
Definition Make.py:4236
Environment(self, path="", bash="", args="", method="source")
Definition Make.py:4325
str postcmd
Command executed during post-build event.
Definition Make.py:4340
str export
Environment variables to be set prior to the execution of the build command.
Definition Make.py:4238
path2exe
(Intel Fortran) Compiler Path
Definition Make.py:4359
str linkcmd
Remote Linker command.
Definition Make.py:4390
ssh_client
Remote workspace.
Definition Make.py:4287
str libname
Name of library, assembled using BuildID.
Definition Make.py:4234
OutputPath(self, libpath, modulepath=None)
Definition Make.py:4267
Base class for all Sphinx build events.
path2exe
Path to Sphinx executable.
Definition Make.py:3788
__init__(self, *args, **kwargs)
Definition Make.py:3770
str exe
Executable of Sphinx.
Definition Make.py:3790
create(self, **kwargs)
Definition Make.py:3889
parse(cls, **kwargs)
Definition Make.py:3821
BuildOption
Ordered dictionary containing all available options.
Definition Make.py:3792
list incdirs
Blank version of list containing library directories without initially specifying MKL.
Definition Make.py:3806
str MakeObjectKind
String identifier of current instance.
Definition Make.py:3776
Settings(self, **kwargs)
Definition Make.py:3810
Abstract meta class for all data class objects.
Class to create 2to3 compatible pickling dictionary.
Class to create 2to3 compatible pickling dictionary.
Module containing all relevant modules and scripts associated with the building process.
Definition __init__.py:1
Module containing basic functionalities defined for convenience.
Definition __init__.py:1