"Fossies" - the Fresh Open Source Software Archive

Member "snapcraft-3.8/snapcraft/internal/project_loader/_config.py" (9 Sep 2019, 14422 Bytes) of package /linux/misc/snapcraft-3.8.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "_config.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 3.7.2_vs_3.8.

    1 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
    2 #
    3 # Copyright (C) 2015-2018 Canonical Ltd
    4 #
    5 # This program is free software: you can redistribute it and/or modify
    6 # it under the terms of the GNU General Public License version 3 as
    7 # published by the Free Software Foundation.
    8 #
    9 # This program is distributed in the hope that it will be useful,
   10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
   11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   12 # GNU General Public License for more details.
   13 #
   14 # You should have received a copy of the GNU General Public License
   15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
   16 
   17 import collections
   18 import logging
   19 import os
   20 import os.path
   21 import re
   22 
   23 import jsonschema
   24 from typing import Set  # noqa: F401
   25 
   26 from snapcraft import project, formatting_utils
   27 from snapcraft.internal import common, deprecations, repo, states, steps
   28 from snapcraft.internal.errors import SnapcraftEnvironmentError
   29 from snapcraft.project._schema import Validator
   30 from ._parts_config import PartsConfig
   31 from ._extensions import apply_extensions
   32 from ._env import (
   33     build_env_for_stage,
   34     runtime_env,
   35     snapcraft_global_environment,
   36     environment_to_replacements,
   37 )
   38 from . import errors, grammar_processing, replace_attr
   39 
   40 
   41 logger = logging.getLogger(__name__)
   42 
   43 
   44 @jsonschema.FormatChecker.cls_checks("icon-path")
   45 def _validate_icon(instance):
   46     allowed_extensions = [".png", ".svg"]
   47     extension = os.path.splitext(instance.lower())[1]
   48     if extension not in allowed_extensions:
   49         raise jsonschema.exceptions.ValidationError(
   50             "'icon' must be either a .png or a .svg"
   51         )
   52 
   53     if not os.path.exists(instance):
   54         raise jsonschema.exceptions.ValidationError(
   55             "Specified icon '{}' does not exist".format(instance)
   56         )
   57 
   58     return True
   59 
   60 
   61 @jsonschema.FormatChecker.cls_checks("epoch", raises=errors.InvalidEpochError)
   62 def _validate_epoch(instance):
   63     str_instance = str(instance)
   64     pattern = re.compile("^(?:0|[1-9][0-9]*[*]?)$")
   65     if not pattern.match(str_instance):
   66         raise errors.InvalidEpochError()
   67 
   68     return True
   69 
   70 
   71 @jsonschema.FormatChecker.cls_checks("architectures")
   72 def _validate_architectures(instance):
   73     standalone_build_ons = collections.Counter()
   74     build_ons = collections.Counter()
   75     run_ons = collections.Counter()
   76 
   77     saw_strings = False
   78     saw_dicts = False
   79 
   80     for item in instance:
   81         # This could either be a dict or a string. In the latter case, the
   82         # schema will take care of it. We just need to further validate the
   83         # dict.
   84         if isinstance(item, str):
   85             saw_strings = True
   86         elif isinstance(item, dict):
   87             saw_dicts = True
   88             build_on = _get_architectures_set(item, "build-on")
   89             build_ons.update(build_on)
   90 
   91             # Add to the list of run-ons. However, if no run-on is specified,
   92             # we know it's implicitly the value of build-on, so use that
   93             # for validation instead.
   94             run_on = _get_architectures_set(item, "run-on")
   95             if run_on:
   96                 run_ons.update(run_on)
   97             else:
   98                 standalone_build_ons.update(build_on)
   99 
  100     # Architectures can either be a list of strings, or a list of objects.
  101     # Mixing the two forms is unsupported.
  102     if saw_strings and saw_dicts:
  103         raise jsonschema.exceptions.ValidationError(
  104             "every item must either be a string or an object",
  105             path=["architectures"],
  106             instance=instance,
  107         )
  108 
  109     # At this point, individual build-ons and run-ons have been validated,
  110     # we just need to validate them across each other.
  111 
  112     # First of all, if we have a `run-on: [all]` (or a standalone
  113     # `build-on: [all]`) then we should only have one item in the instance,
  114     # otherwise we know we'll have multiple snaps claiming they run on the same
  115     # architectures (i.e. all and something else).
  116     number_of_snaps = len(instance)
  117     if "all" in run_ons and number_of_snaps > 1:
  118         raise jsonschema.exceptions.ValidationError(
  119             "one of the items has 'all' in 'run-on', but there are {} "
  120             "items: upon release they will conflict. 'all' should only be "
  121             "used if there is a single item".format(number_of_snaps),
  122             path=["architectures"],
  123             instance=instance,
  124         )
  125     if "all" in build_ons and number_of_snaps > 1:
  126         raise jsonschema.exceptions.ValidationError(
  127             "one of the items has 'all' in 'build-on', but there are {} "
  128             "items: snapcraft doesn't know which one to use. 'all' should "
  129             "only be used if there is a single item".format(number_of_snaps),
  130             path=["architectures"],
  131             instance=instance,
  132         )
  133 
  134     # We want to ensure that multiple `run-on`s (or standalone `build-on`s)
  135     # don't include the same arch, or they'll clash with each other when
  136     # releasing.
  137     all_run_ons = run_ons + standalone_build_ons
  138     duplicates = {arch for (arch, count) in all_run_ons.items() if count > 1}
  139     if duplicates:
  140         raise jsonschema.exceptions.ValidationError(
  141             "multiple items will build snaps that claim to run on {}".format(
  142                 formatting_utils.humanize_list(duplicates, "and")
  143             ),
  144             path=["architectures"],
  145             instance=instance,
  146         )
  147 
  148     # Finally, ensure that multiple `build-on`s don't include the same arch
  149     # or Snapcraft has no way of knowing which one to use.
  150     duplicates = {arch for (arch, count) in build_ons.items() if count > 1}
  151     if duplicates:
  152         raise jsonschema.exceptions.ValidationError(
  153             "{} {} present in the 'build-on' of multiple items, which means "
  154             "snapcraft doesn't know which 'run-on' to use when building on "
  155             "{} {}".format(
  156                 formatting_utils.humanize_list(duplicates, "and"),
  157                 formatting_utils.pluralize(duplicates, "is", "are"),
  158                 formatting_utils.pluralize(duplicates, "that", "those"),
  159                 formatting_utils.pluralize(duplicates, "architecture", "architectures"),
  160             ),
  161             path=["architectures"],
  162             instance=instance,
  163         )
  164 
  165     return True
  166 
  167 
  168 def _get_architectures_set(item, name):
  169     value = item.get(name, set())
  170     if isinstance(value, str):
  171         value_set = {value}
  172     else:
  173         value_set = set(value)
  174 
  175     _validate_architectures_set(value_set, name)
  176 
  177     return value_set
  178 
  179 
  180 def _validate_architectures_set(architectures_set, name):
  181     if "all" in architectures_set and len(architectures_set) > 1:
  182         raise jsonschema.exceptions.ValidationError(
  183             "'all' can only be used within {!r} by itself, "
  184             "not with other architectures".format(name),
  185             path=["architectures"],
  186             instance=architectures_set,
  187         )
  188 
  189 
  190 class Config:
  191     @property
  192     def part_names(self):
  193         return self.parts.part_names
  194 
  195     @property
  196     def all_parts(self):
  197         return self.parts.all_parts
  198 
  199     def __init__(self, project: project.Project) -> None:
  200         self.build_snaps = set()  # type: Set[str]
  201         self.project = project
  202 
  203         # raw_snapcraft_yaml is read only, create a new copy
  204         snapcraft_yaml = apply_extensions(project.info.get_raw_snapcraft())
  205 
  206         self.validator = Validator(snapcraft_yaml)
  207         self.validator.validate()
  208 
  209         snapcraft_yaml = self._expand_filesets(snapcraft_yaml)
  210 
  211         self.data = self._expand_env(snapcraft_yaml)
  212         self._ensure_no_duplicate_app_aliases()
  213 
  214         grammar_processor = grammar_processing.GlobalGrammarProcessor(
  215             properties=self.data, project=project
  216         )
  217 
  218         self.build_tools = grammar_processor.get_build_packages()
  219         self.build_tools |= set(project.additional_build_packages)
  220 
  221         # If version: git is used we want to add "git" to build-packages
  222         if self.data.get("version") == "git":
  223             self.build_tools.add("git")
  224 
  225         # Always add the base for building for non os and base snaps
  226         if project.info.base is None and project.info.type in ("app", "gadget"):
  227             raise SnapcraftEnvironmentError(
  228                 "A base is required for snaps of type {!r}.".format(project.info.type)
  229             )
  230         if project.info.base is not None:
  231             # If the base is already installed by other means, skip its installation.
  232             # But, we should always add it when in a docker environment so
  233             # the creator of said docker image is aware that it is required.
  234             if common.is_process_container() or not repo.snaps.SnapPackage.is_snap_installed(
  235                 project.info.base
  236             ):
  237                 self.build_snaps.add(project.info.base)
  238 
  239         self.parts = PartsConfig(
  240             parts=self.data,
  241             project=project,
  242             validator=self.validator,
  243             build_snaps=self.build_snaps,
  244             build_tools=self.build_tools,
  245         )
  246 
  247         self.data["architectures"] = _process_architectures(
  248             self.data.get("architectures"), project.deb_arch
  249         )
  250 
  251     def _ensure_no_duplicate_app_aliases(self):
  252         # Prevent multiple apps within a snap from having duplicate alias names
  253         aliases = []
  254         for app_name, app in self.data.get("apps", {}).items():
  255             aliases.extend(app.get("aliases", []))
  256 
  257         # The aliases property is actually deprecated:
  258         if aliases:
  259             deprecations.handle_deprecation_notice("dn5")
  260         seen = set()
  261         duplicates = set()
  262         for alias in aliases:
  263             if alias in seen:
  264                 duplicates.add(alias)
  265             else:
  266                 seen.add(alias)
  267         if duplicates:
  268             raise errors.DuplicateAliasError(aliases=duplicates)
  269 
  270     def get_project_state(self, step: steps.Step):
  271         """Returns a dict of states for the given step of each part."""
  272 
  273         state = {}
  274         for part in self.parts.all_parts:
  275             state[part.name] = states.get_state(part.plugin.statedir, step)
  276 
  277         return state
  278 
  279     def stage_env(self):
  280         stage_dir = self.project.stage_dir
  281         env = []
  282 
  283         env += runtime_env(stage_dir, self.project.arch_triplet)
  284         env += build_env_for_stage(
  285             stage_dir, self.data["name"], self.project.arch_triplet
  286         )
  287         for part in self.parts.all_parts:
  288             env += part.env(stage_dir)
  289 
  290         return env
  291 
  292     def snap_env(self):
  293         prime_dir = self.project.prime_dir
  294         env = []
  295 
  296         env += runtime_env(prime_dir, self.project.arch_triplet)
  297         dependency_paths = set()
  298         for part in self.parts.all_parts:
  299             env += part.env(prime_dir)
  300             dependency_paths |= part.get_primed_dependency_paths()
  301 
  302         # Dependency paths are only valid if they actually exist. Sorting them
  303         # here as well so the LD_LIBRARY_PATH is consistent between runs.
  304         dependency_paths = sorted(
  305             {path for path in dependency_paths if os.path.isdir(path)}
  306         )
  307 
  308         if dependency_paths:
  309             # Add more specific LD_LIBRARY_PATH from the dependencies.
  310             env.append(
  311                 'LD_LIBRARY_PATH="' + ":".join(dependency_paths) + ':$LD_LIBRARY_PATH"'
  312             )
  313 
  314         return env
  315 
  316     def project_env(self):
  317         return [
  318             '{}="{}"'.format(variable, value)
  319             for variable, value in snapcraft_global_environment(self.project).items()
  320         ]
  321 
  322     def _expand_env(self, snapcraft_yaml):
  323         environment_keys = ["name", "version"]
  324         for key in snapcraft_yaml:
  325             if any((key == env_key for env_key in environment_keys)):
  326                 continue
  327 
  328             replacements = environment_to_replacements(
  329                 snapcraft_global_environment(self.project)
  330             )
  331 
  332             snapcraft_yaml[key] = replace_attr(snapcraft_yaml[key], replacements)
  333         return snapcraft_yaml
  334 
  335     def _expand_filesets(self, snapcraft_yaml):
  336         parts = snapcraft_yaml.get("parts", {})
  337 
  338         for part_name in parts:
  339             for step in ("stage", "prime"):
  340                 step_fileset = _expand_filesets_for(step, parts[part_name])
  341                 parts[part_name][step] = step_fileset
  342 
  343         return snapcraft_yaml
  344 
  345 
  346 def _expand_filesets_for(step, properties):
  347     filesets = properties.get("filesets", {})
  348     fileset_for_step = properties.get(step, {})
  349     new_step_set = []
  350 
  351     for item in fileset_for_step:
  352         if item.startswith("$"):
  353             try:
  354                 new_step_set.extend(filesets[item[1:]])
  355             except KeyError:
  356                 raise errors.SnapcraftLogicError(
  357                     "'{}' referred to in the '{}' fileset but it is not "
  358                     "in filesets".format(item, step)
  359                 )
  360         else:
  361             new_step_set.append(item)
  362 
  363     return new_step_set
  364 
  365 
  366 class _Architecture:
  367     def __init__(self, *, build_on, run_on=None):
  368         if isinstance(build_on, str):
  369             self.build_on = [build_on]
  370         else:
  371             self.build_on = build_on
  372 
  373         # If there is no run_on, it defaults to the value of build_on
  374         if not run_on:
  375             self.run_on = self.build_on
  376         elif isinstance(run_on, str):
  377             self.run_on = [run_on]
  378         else:
  379             self.run_on = run_on
  380 
  381 
  382 def _create_architecture_list(architectures, current_arch):
  383     if not architectures:
  384         return [_Architecture(build_on=[current_arch])]
  385 
  386     build_architectures = []  # type: List[str]
  387     architecture_list = []  # type: List[_Architecture]
  388     for item in architectures:
  389         if isinstance(item, str):
  390             build_architectures.append(item)
  391         if isinstance(item, dict):
  392             architecture_list.append(
  393                 _Architecture(build_on=item.get("build-on"), run_on=item.get("run-on"))
  394             )
  395 
  396     if build_architectures:
  397         architecture_list.append(_Architecture(build_on=build_architectures))
  398 
  399     return architecture_list
  400 
  401 
  402 def _process_architectures(architectures, current_arch):
  403     architecture_list = _create_architecture_list(architectures, current_arch)
  404 
  405     for architecture in architecture_list:
  406         if current_arch in architecture.build_on or "all" in architecture.build_on:
  407             return architecture.run_on
  408 
  409     return [current_arch]