3 # Copyright (C) 2010-2013 CEA/DEN
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 In this file : all functions that do a system call,
21 like open a browser or an editor, or call a git command
25 import subprocess as SP
32 import utilsSat as UTS
35 from . import printcolors
37 def show_in_editor(editor, filePath, logger):
38 '''open filePath using editor.
40 :param editor str: The editor to use.
41 :param filePath str: The path to the file to open.
43 # default editor is vi
44 if editor is None or len(editor) == 0:
47 if '%s' not in editor:
51 # launch cmd using subprocess.Popen
52 cmd = editor % filePath
53 logger.write('Launched command:\n' + cmd + '\n', 5)
54 p = SP.Popen(cmd, shell=True)
57 logger.write(printcolors.printcError(_("Unable to edit file %s\n")
60 def show_in_webbrowser(editor, filePath, logger):
61 '''open filePath using web browser firefox, chromium etc...
62 if file is xml, previous http sever is done before to fix new security problems
64 :param editor str: The web browser to use.
65 :param filePath str: The path to the file to open.
68 # default editor is firefox
69 if editor is None or len(editor) == 0:
72 path, namefile = os.path.split(filePath)
73 basefile, ext = os.path.splitext(namefile)
75 # previouly http.server 8765/6/7... kill ... or not ? TODO wait and see REX
76 port = os.getenv('SAT_PORT_LOG', '8765')
77 for proc in psutil.process_iter():
79 cmdline = " ".join(proc.cmdline())
80 if "python3 -m http.server %s" % port in cmdline:
81 print("kill previous process '%s'" % cmdline)
82 proc.kill() # TODO may be not owner ? -> change 8766/7/8... as SAT_PORT_LOG
87 python3 -m http.server %(port)s &> /dev/null &
88 %(editor)s http://localhost:%(port)s/%(namefile)s
89 """ % {"path": path, "editor": editor, "namefile": namefile, 'port': port}
91 # print("show_in_webbrowser:\n%s" % cmd)
94 # launch cmd using subprocess.Popen
95 logger.write('Launched command:\n%s\n' % cmd, 5)
96 p = SP.Popen(cmd, shell=True, stdout=SP.PIPE, stderr=SP.STDOUT)
97 res_out, _ = p.communicate() # _ = None as stderr=SP.STDOUT
98 # print("Launched command stdout:\n%s" % res_out)
99 except Exception as e:
100 logger.write(printcolors.printcError(_("Unable to display file %s\n%s\n")
104 def git_describe(repo_path):
105 '''Use git describe --tags command to return tag description of the git repository"
106 :param repo_path str: The git repository to describe
108 git_cmd="cd %s;git describe --tags" % repo_path
109 p = SP.Popen(git_cmd, shell=True, stdin=SP.PIPE, stdout=SP.PIPE, stderr=SP.PIPE)
111 if p.returncode != 0:
114 tag_description=p.stdout.readlines()[0].strip()
115 # with python3 this utf8 bytes should be decoded
116 if isinstance(tag_description, bytes):
117 tag_description=tag_description.decode("utf-8", "ignore")
118 return tag_description
121 def git_extract(from_what, tag, git_options, where, logger, environment=None):
122 '''Extracts sources from a git repository.
124 :param from_what str: The remote git repository.
125 :param tag str: The tag.
126 :param git_options str: git options
127 :param where str: The path where to extract.
128 :param logger Logger: The logger instance to use.
129 :param environment src.environment.Environ: The environment to source when extracting.
130 :return: True if the extraction is successful
133 DBG.write("git_extract", [from_what, tag, str(where)])
134 if not where.exists():
136 where_git = os.path.join(str(where), ".git")
137 if tag == "master" or tag == "HEAD":
138 if src.architecture.is_windows():
139 cmd = "git clone %(git_options)s %(remote)s %(where)s"
143 git clone %(git_options)s %(remote)s %(where)s
145 if [ $res -eq 0 ]; then
146 touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s
150 cmd = cmd % {'git_options': git_options, 'remote': from_what, 'tag': tag, 'where': str(where), 'where_git': where_git}
152 # NOTICE: this command only works with recent version of git
153 # because --work-tree does not work with an absolute path
154 if src.architecture.is_windows():
155 cmd = "rmdir /S /Q %(where)s && git clone %(git_options)s %(remote)s %(where)s && git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s"
157 # for sat compile --update : changes the date of directory, only for branches, not tag
161 git clone %(git_options)s %(remote)s %(where)s && \
162 git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s
164 git --git-dir=%(where_git)s status | grep HEAD
165 if [ $res -eq 0 -a $? -ne 0 ]; then
166 touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s
170 cmd = cmd % {'git_options': git_options,
174 'where_git': where_git}
177 cmd=cmd.replace('date_format', '"%ai"')
178 logger.logTxtFile.write("\n" + cmd + "\n")
179 logger.logTxtFile.flush()
181 DBG.write("cmd", cmd)
182 # git commands may fail sometimes for various raisons
183 # (big module, network troubles, tuleap maintenance)
184 # therefore we give several tries
186 max_number_of_tries = 3
187 sleep_delay = 30 # seconds
190 rc = UTS.Popen(cmd, cwd=str(where.dir()), env=environment.environ.environ, logger=logger)
191 if rc.isOk() or (i_try>=max_number_of_tries):
193 logger.write('\ngit command failed! Wait %d seconds and give an other try (%d/%d)\n' % \
194 (sleep_delay, i_try + 1, max_number_of_tries), 3)
195 time.sleep(sleep_delay) # wait a little
200 def git_extract_sub_dir(from_what, tag, git_options, where, sub_dir, logger, environment=None):
201 '''Extracts sources from a subtree sub_dir of a git repository.
203 :param from_what str: The remote git repository.
204 :param tag str: The tag.
205 :param git_options str: git options
206 :param where str: The path where to extract.
207 :param sub_dir str: The relative path of subtree to extract.
208 :param logger Logger: The logger instance to use.
209 :param environment src.environment.Environ: The environment to source when extracting.
210 :return: True if the extraction is successful
213 strWhere = str(where)
214 tmpWhere = strWhere + '_tmp'
215 parentWhere = os.path.dirname(strWhere)
216 if not os.path.exists(parentWhere):
217 logger.error("not existing directory: %s" % parentWhere)
219 if os.path.isdir(strWhere):
220 logger.error("do not override existing directory: %s" % strWhere)
222 aDict = {'git_options': git_options,
227 'parentWhere': parentWhere,
228 'tmpWhere': tmpWhere,
230 DBG.write("git_extract_sub_dir", aDict)
231 if not src.architecture.is_windows():
234 export tmpDir=%(tmpWhere)s && \
236 git clone %(git_options)s %(remote)s $tmpDir && \
238 git checkout %(tag)s && \
239 mv %(sub_dir)s %(where)s && \
240 git log -1 > %(where)s/README_git_log.txt && \
246 set tmpDir=%(tmpWhere)s && \
248 git clone %(git_options)s %(remote)s %tmpDir% && \
250 git checkout %(tag)s && \
251 mv %(sub_dir)s %(where)s && \
252 git log -1 > %(where)s\\README_git_log.txt && \
256 DBG.write("cmd", cmd)
258 for nbtry in range(0,3): # retries case of network problem
259 rc = UTS.Popen(cmd, cwd=parentWhere, env=environment.environ.environ, logger=logger)
261 time.sleep(30) # wait a little
265 def archive_extract(from_what, where, logger):
266 '''Extracts sources from an archive.
268 :param from_what str: The path to the archive.
269 :param where str: The path where to extract.
270 :param logger Logger: The logger instance to use.
271 :return: True if the extraction is successful
275 archive = tarfile.open(from_what)
276 for i in archive.getmembers():
277 archive.extract(i, path=str(where))
278 return True, os.path.commonprefix(archive.getnames())
279 except Exception as exc:
280 logger.write("archive_extract: %s\n" % exc)
283 def cvs_extract(protocol, user, server, base, tag, product, where,
284 logger, checkout=False, environment=None):
285 '''Extracts sources from a cvs repository.
287 :param protocol str: The cvs protocol.
288 :param user str: The user to be used.
289 :param server str: The remote cvs server.
291 :param tag str: The tag.
292 :param product str: The product.
293 :param where str: The path where to extract.
294 :param logger Logger: The logger instance to use.
295 :param checkout boolean: If true use checkout cvs.
296 :param environment src.environment.Environ: The environment to source when
298 :return: True if the extraction is successful
303 if tag is not None and len(tag) > 0:
309 elif len(opttag) == 0:
312 if len(protocol) > 0:
313 root = "%s@%s:%s" % (user, server, base)
314 command = "cvs -d :%(protocol)s:%(root)s %(command)s -d %(where)s %(tag)s %(product)s" % \
315 { 'protocol': protocol, 'root': root, 'where': str(where.base()),
316 'tag': opttag, 'product': product, 'command': cmd }
318 command = "cvs -d %(root)s %(command)s -d %(where)s %(tag)s %(base)s/%(product)s" % \
319 { 'root': server, 'base': base, 'where': str(where.base()),
320 'tag': opttag, 'product': product, 'command': cmd }
322 logger.write(command + "\n", 5)
324 if not where.dir().exists():
327 logger.logTxtFile.write("\n" + command + "\n")
328 logger.logTxtFile.flush()
329 res = SP.call(command, cwd=str(where.dir()),
330 env=environment.environ.environ,
332 stdout=logger.logTxtFile,
336 def svn_extract(user,
343 '''Extracts sources from a svn repository.
345 :param user str: The user to be used.
346 :param from_what str: The remote git repository.
347 :param tag str: The tag.
348 :param where str: The path where to extract.
349 :param logger Logger: The logger instance to use.
350 :param checkout boolean: If true use checkout svn.
351 :param environment src.environment.Environ: The environment to source when
353 :return: True if the extraction is successful
356 if not where.exists():
360 command = "svn checkout --username %(user)s %(remote)s %(where)s" % \
361 { 'remote': from_what, 'user' : user, 'where': str(where) }
364 if os.path.exists(str(where)):
365 command = "/bin/rm -rf %(where)s && " % \
366 { 'remote': from_what, 'where': str(where) }
369 command += "svn export --username %(user)s %(remote)s %(where)s" % \
370 { 'remote': from_what, 'user' : user, 'where': str(where) }
372 command += "svn export -r %(tag)s --username %(user)s %(remote)s %(where)s" % \
373 { 'tag' : tag, 'remote': from_what, 'user' : user, 'where': str(where) }
375 logger.logTxtFile.write(command + "\n")
377 logger.write(command + "\n", 5)
378 logger.logTxtFile.write("\n" + command + "\n")
379 logger.logTxtFile.flush()
380 res = SP.call(command, cwd=str(where.dir()),
381 env=environment.environ.environ,
383 stdout=logger.logTxtFile,
387 def get_pkg_check_cmd(dist_name):
388 '''Build the command to use for checking if a linux package is installed or not.'''
390 if dist_name in ["CO","FD","MG","MD","CO","OS"]: # linux using rpm
392 manager_msg_err="Error : command failed because sat was not able to find apt command"
395 manager_msg_err="Error : command failed because sat was not able to find rpm command"
397 # 1- search for an installed package manager (rpm on rh, apt or dpkg-query on db)
398 cmd_which_rpm = ["which", "rpm"]
399 cmd_which_apt = ["which", "apt"]
400 cmd_which_dpkg = ["which", "dpkg-query"]
401 with open(os.devnull, 'w') as devnull:
402 # 1) we search for apt (debian based systems)
403 completed=SP.call(cmd_which_dpkg,stdout=devnull, stderr=SP.STDOUT)
404 if completed==0 and linux=="DB":
405 cmd_is_package_installed = ["dpkg-query", "-l"]
407 # 2) if dpkg not found search for apt
408 completed = SP.call(cmd_which_apt, stdout=devnull, stderr=SP.STDOUT)
409 if completed == 0 and linux == "DB":
410 cmd_is_package_installed = ["apt", "list", "--installed"]
412 # 3) if apt not found search for rpm (redhat)
413 completed=SP.call(cmd_which_rpm,stdout=devnull, stderr=SP.STDOUT) # only 3.8! ,capture_output=True)
414 if completed==0 and linux=="RH":
415 cmd_is_package_installed=["rpm", "-q"]
417 # no package manager was found corresponding to dist_name
418 raise src.SatException(manager_msg_err)
419 return cmd_is_package_installed
421 def check_system_pkg(check_cmd,pkg):
422 '''Check if a package is installed
423 :param check_cmd list: the list of command to use system package manager
424 :param user str: the pkg name to check
426 :return: a string with package name with status un message
429 FNULL = open(os.devnull, 'w')
430 cmd_is_package_installed=[]
431 for cmd in check_cmd:
432 cmd_is_package_installed.append(cmd)
433 cmd_is_package_installed.append(pkg)
436 if check_cmd[0]=="apt":
437 # special treatment for apt
438 # apt output is too messy for being used
439 # some debian packages have version numbers in their name, we need to add a *
440 # also apt do not return status, we need to use grep
441 # and apt output is too messy for being used
442 cmd_is_package_installed[-1]+="*" # we don't specify in pyconf the exact name because of version numbers
443 p=SP.Popen(cmd_is_package_installed, stdout=SP.PIPE, stderr=FNULL)
445 output = SP.check_output(['grep', pkg], stdin=p.stdout)
446 msg_status=src.printcolors.printcSuccess("OK")
448 msg_status=src.printcolors.printcError("KO")
449 msg_status+=" (package is not installed!)\n"
450 elif check_cmd[0] == "dpkg-query":
451 # special treatment for dpkg-query
452 # some debian packages have version numbers in their name, we need to add a *
453 # also dpkg-query do not return status, we need to use grep
454 # and dpkg-query output is too messy for being used
455 cmd_is_package_installed[-1] = (
456 cmd_is_package_installed[-1] + "*"
457 ) # we don't specify in pyconf the exact name because of version numbers
458 p = SP.Popen(cmd_is_package_installed, stdout=SP.PIPE, stderr=FNULL)
460 output = SP.check_output(["grep", "^ii"], stdin=p.stdout)
461 msg_status = src.printcolors.printcSuccess("OK")
462 except SP.CalledProcessError:
463 msg_status = src.printcolors.printcError("KO")
464 msg_status += " (package is not installed!)\n"
466 p=SP.Popen(cmd_is_package_installed, stdout=SP.PIPE, stderr=FNULL)
467 output, err = p.communicate()
470 msg_status=src.printcolors.printcSuccess("OK")
471 # in python3 output is a byte and should be decoded
472 if isinstance(output, bytes):
473 output = output.decode("utf-8", "ignore")
474 msg_status+=" (" + output.replace('\n',' ') + ")\n" # remove output trailing \n
476 msg_status=src.printcolors.printcError("KO")
477 msg_status+=" (package is not installed!)\n"