"Fossies" - the Fresh Open Source Software Archive

Member "solr-8.4.1/dev-tools/scripts/releaseWizard.py" (10 Jan 2020, 78528 Bytes) of package /linux/www/solr-8.4.1-src.tgz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "releaseWizard.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 8.4.0_vs_8.4.1.

    1 #!/usr/bin/env python3
    2 # -*- coding: utf-8 -*-
    3 # Licensed to the Apache Software Foundation (ASF) under one or more
    4 # contributor license agreements.  See the NOTICE file distributed with
    5 # this work for additional information regarding copyright ownership.
    6 # The ASF licenses this file to You under the Apache License, Version 2.0
    7 # (the "License"); you may not use this file except in compliance with
    8 # the License.  You may obtain a copy of the License at
    9 #
   10 #     http://www.apache.org/licenses/LICENSE-2.0
   11 #
   12 # Unless required by applicable law or agreed to in writing, software
   13 # distributed under the License is distributed on an "AS IS" BASIS,
   14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   15 # See the License for the specific language governing permissions and
   16 # limitations under the License.
   17 
   18 # This script is a wizard that aims to (some day) replace the todoList at https://wiki.apache.org/lucene-java/ReleaseTodo
   19 # It will walk you through the steps of the release process, asking for decisions or input along the way
   20 # CAUTION: This is an alpha version, please read the HELP section in the main menu.
   21 #
   22 # Requirements:
   23 #   Install requirements with this command:
   24 #   pip3 install -r requirements.txt
   25 #
   26 # Usage:
   27 #   releaseWizard.py [-h] [--dry-run] [--root PATH]
   28 #
   29 #   optional arguments:
   30 #   -h, --help   show this help message and exit
   31 #   --dry-run    Do not execute any commands, but echo them instead. Display
   32 #   extra debug info
   33 #   --root PATH  Specify different root folder than ~/.lucene-releases
   34 
   35 import argparse
   36 import copy
   37 import fcntl
   38 import json
   39 import os
   40 import platform
   41 import re
   42 import shlex
   43 import shutil
   44 import subprocess
   45 import sys
   46 import textwrap
   47 import time
   48 import urllib
   49 from collections import OrderedDict
   50 from datetime import datetime
   51 from datetime import timedelta
   52 
   53 try:
   54     import holidays
   55     import yaml
   56     from ics import Calendar, Event
   57     from jinja2 import Environment
   58 except:
   59     print("You lack some of the module dependencies to run this script.")
   60     print("Please run 'pip3 install -r requirements.txt' and try again.")
   61     sys.exit(1)
   62 
   63 import scriptutil
   64 from consolemenu import ConsoleMenu
   65 from consolemenu.items import FunctionItem, SubmenuItem, ExitItem
   66 from consolemenu.screen import Screen
   67 from scriptutil import BranchType, Version, check_ant, download, run
   68 
   69 # Solr-to-Java version mapping
   70 java_versions = {6: 8, 7: 8, 8: 8, 9: 11}
   71 
   72 
   73 # Edit this to add other global jinja2 variables or filters
   74 def expand_jinja(text, vars=None):
   75     global_vars = OrderedDict({
   76         'script_version': state.script_version,
   77         'release_version': state.release_version,
   78         'release_version_underscore': state.release_version.replace('.', '_'),
   79         'release_date': state.get_release_date(),
   80         'ivy2_folder': os.path.expanduser("~/.ivy2/"),
   81         'config_path': state.config_path,
   82         'rc_number': state.rc_number,
   83         'script_branch': state.script_branch,
   84         'release_folder': state.get_release_folder(),
   85         'git_checkout_folder': state.get_git_checkout_folder(),
   86         'dist_url_base': 'https://dist.apache.org/repos/dist/dev/lucene',
   87         'm2_repository_url': 'https://repository.apache.org/service/local/staging/deploy/maven2',
   88         'dist_file_path': state.get_dist_folder(),
   89         'rc_folder': state.get_rc_folder(),
   90         'base_branch': state.get_base_branch_name(),
   91         'release_branch': state.release_branch,
   92         'stable_branch': state.get_stable_branch_name(),
   93         'minor_branch': state.get_minor_branch_name(),
   94         'release_type': state.release_type,
   95         'is_feature_release': state.release_type in ['minor', 'major'],
   96         'release_version_major': state.release_version_major,
   97         'release_version_minor': state.release_version_minor,
   98         'release_version_bugfix': state.release_version_bugfix,
   99         'state': state,
  100         'epoch': unix_time_millis(datetime.utcnow()),
  101         'get_next_version': state.get_next_version(),
  102         'current_git_rev': state.get_current_git_rev(),
  103         'keys_downloaded': keys_downloaded(),
  104         'editor': get_editor(),
  105         'rename_cmd': 'ren' if is_windows() else 'mv',
  106         'vote_close_72h': vote_close_72h_date().strftime("%Y-%m-%d %H:00 UTC"),
  107         'vote_close_72h_epoch': unix_time_millis(vote_close_72h_date()),
  108         'vote_close_72h_holidays': vote_close_72h_holidays(),
  109         'lucene_highlights_file': lucene_highlights_file,
  110         'solr_highlights_file': solr_highlights_file,
  111         'tlp_news_draft': tlp_news_draft,
  112         'lucene_news_draft': lucene_news_draft,
  113         'solr_news_draft': solr_news_draft,
  114         'tlp_news_file': tlp_news_file,
  115         'lucene_news_file': lucene_news_file,
  116         'solr_news_file': solr_news_file,
  117         'load_lines': load_lines,
  118         'set_java_home': set_java_home,
  119         'latest_version': state.get_latest_version(),
  120         'latest_lts_version': state.get_latest_lts_version(),
  121         'master_version': state.get_master_version(),
  122         'mirrored_versions': state.get_mirrored_versions(),
  123         'mirrored_versions_to_delete': state.get_mirrored_versions_to_delete(),
  124         'home': os.path.expanduser("~")
  125     })
  126     global_vars.update(state.get_todo_states())
  127     if vars:
  128         global_vars.update(vars)
  129 
  130     filled = replace_templates(text)
  131 
  132     try:
  133         env = Environment(lstrip_blocks=True, keep_trailing_newline=False, trim_blocks=True)
  134         env.filters['path_join'] = lambda paths: os.path.join(*paths)
  135         env.filters['expanduser'] = lambda path: os.path.expanduser(path)
  136         env.filters['formatdate'] = lambda date: (datetime.strftime(date, "%-d %B %Y") if date else "<date>" )
  137         template = env.from_string(str(filled), globals=global_vars)
  138         filled = template.render()
  139     except Exception as e:
  140         print("Exception while rendering jinja template %s: %s" % (str(filled)[:10], e))
  141     return filled
  142 
  143 
  144 def replace_templates(text):
  145     tpl_lines = []
  146     for line in text.splitlines():
  147         if line.startswith("(( template="):
  148             match = re.search(r"^\(\( template=(.+?) \)\)", line)
  149             name = match.group(1)
  150             tpl_lines.append(replace_templates(templates[name].strip()))
  151         else:
  152             tpl_lines.append(line)
  153     return "\n".join(tpl_lines)
  154 
  155 
  156 def getScriptVersion():
  157     topLevelDir = os.path.join(os.path.abspath("%s/" % script_path), os.path.pardir, os.path.pardir)
  158     reBaseVersion = re.compile(r'version\.base\s*=\s*(\d+\.\d+\.\d+)')
  159     return reBaseVersion.search(open('%s/lucene/version.properties' % topLevelDir).read()).group(1)
  160 
  161 
  162 def get_editor():
  163     return os.environ['EDITOR'] if 'EDITOR' in os.environ else 'notepad.exe' if is_windows() else 'vi'
  164 
  165 
  166 def check_prerequisites(todo=None):
  167     if sys.version_info < (3, 4):
  168         sys.exit("Script requires Python v3.4 or later")
  169     try:
  170         gpg_ver = run("gpg --version").splitlines()[0]
  171     except:
  172         sys.exit("You will need gpg installed")
  173     if not check_ant().startswith('1.8'):
  174         print("WARNING: This script will work best with ant 1.8. The script buildAndPushRelease.py may have problems with PGP password input under ant 1.10")
  175     if not 'JAVA8_HOME' in os.environ or not 'JAVA11_HOME' in os.environ:
  176         sys.exit("Please set environment variables JAVA8_HOME and JAVA11_HOME")
  177     try:
  178         asciidoc_ver = run("asciidoctor -V").splitlines()[0]
  179     except:
  180         asciidoc_ver = ""
  181         print("WARNING: In order to export asciidoc version to HTML, you will need asciidoctor installed")
  182     try:
  183         git_ver = run("git --version").splitlines()[0]
  184     except:
  185         sys.exit("You will need git installed")
  186     if not 'EDITOR' in os.environ:
  187         print("WARNING: Environment variable $EDITOR not set, using %s" % get_editor())
  188 
  189     if todo:
  190         print("%s\n%s\n%s\n" % (gpg_ver, asciidoc_ver, git_ver))
  191     return True
  192 
  193 
  194 epoch = datetime.utcfromtimestamp(0)
  195 
  196 
  197 def unix_time_millis(dt):
  198     return int((dt - epoch).total_seconds() * 1000.0)
  199 
  200 
  201 def bootstrap_todos(todo_list):
  202     # Establish links from commands to to_do for finding todo vars
  203     for tg in todo_list:
  204         if dry_run:
  205             print("Group %s" % tg.id)
  206         for td in tg.get_todos():
  207             if dry_run:
  208                 print("  Todo %s" % td.id)
  209             cmds = td.commands
  210             if cmds:
  211                 if dry_run:
  212                     print("  Commands")
  213                 cmds.todo_id = td.id
  214                 for cmd in cmds.commands:
  215                     if dry_run:
  216                         print("    Command %s" % cmd.cmd)
  217                     cmd.todo_id = td.id
  218 
  219     print("Loaded TODO definitions from releaseWizard.yaml")
  220     return todo_list
  221 
  222 
  223 def maybe_remove_rc_from_svn():
  224     todo = state.get_todo_by_id('import_svn')
  225     if todo and todo.is_done():
  226         print("import_svn done")
  227         Commands(state.get_git_checkout_folder(),
  228                  """Looks like you uploaded artifacts for {{ build_rc.git_rev | default("<git_rev>", True) }} to svn which needs to be removed.""",
  229                  [Command(
  230                  """svn -m "Remove cancelled Lucene/Solr {{ release_version }} RC{{ rc_number }}" rm {{ dist_url }}""",
  231                  logfile="svn_rm.log",
  232                  tee=True,
  233                  vars={
  234                      'dist_folder': """lucene-solr-{{ release_version }}-RC{{ rc_number }}-rev{{ build_rc.git_rev | default("<git_rev>", True) }}""",
  235                      'dist_url': "{{ dist_url_base }}/{{ dist_folder }}"
  236                  }
  237              )],
  238                  enable_execute=True, confirm_each_command=False).run()
  239 
  240 
  241 # To be able to hide fields when dumping Yaml
  242 class SecretYamlObject(yaml.YAMLObject):
  243     hidden_fields = []
  244     @classmethod
  245     def to_yaml(cls,dumper,data):
  246         print("Dumping object %s" % type(data))
  247 
  248         new_data = copy.deepcopy(data)
  249         for item in cls.hidden_fields:
  250             if item in new_data.__dict__:
  251                 del new_data.__dict__[item]
  252         for item in data.__dict__:
  253             if item in new_data.__dict__ and new_data.__dict__[item] is None:
  254                 del new_data.__dict__[item]
  255         return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls,
  256                                             flow_style=cls.yaml_flow_style)
  257 
  258 
  259 def str_presenter(dumper, data):
  260     if len(data.split('\n')) > 1:  # check for multiline string
  261         return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
  262     return dumper.represent_scalar('tag:yaml.org,2002:str', data)
  263 
  264 
  265 class ReleaseState:
  266     def __init__(self, config_path, release_version, script_version):
  267         self.script_version = script_version
  268         self.config_path = config_path
  269         self.todo_groups = None
  270         self.todos = None
  271         self.latest_version = None
  272         self.previous_rcs = {}
  273         self.rc_number = 1
  274         self.start_date = unix_time_millis(datetime.utcnow())
  275         self.script_branch = run("git rev-parse --abbrev-ref HEAD").strip()
  276         self.mirrored_versions = None
  277         try:
  278             self.script_branch_type = scriptutil.find_branch_type()
  279         except:
  280             print("WARNING: This script shold (ideally) run from the release branch, not a feature branch (%s)" % self.script_branch)
  281             self.script_branch_type = 'feature'
  282         self.set_release_version(release_version)
  283 
  284     def set_release_version(self, version):
  285         self.validate_release_version(self.script_branch_type, self.script_branch, version)
  286         self.release_version = version
  287         v = Version.parse(version)
  288         self.release_version_major = v.major
  289         self.release_version_minor = v.minor
  290         self.release_version_bugfix = v.bugfix
  291         self.release_branch = "branch_%s_%s" % (v.major, v.minor)
  292         if v.is_major_release():
  293             self.release_type = 'major'
  294         elif v.is_minor_release():
  295             self.release_type = 'minor'
  296         else:
  297             self.release_type = 'bugfix'
  298 
  299     def is_released(self):
  300         return self.get_todo_by_id('announce_lucene').is_done()
  301 
  302     def get_release_date(self):
  303         publish_task = self.get_todo_by_id('publish_maven')
  304         if publish_task.is_done():
  305             return unix_to_datetime(publish_task.get_state()['done_date'])
  306         else:
  307             return None
  308 
  309     def get_latest_version(self):
  310         if self.latest_version is None:
  311             versions = self.get_mirrored_versions()
  312             latest = versions[0]
  313             for ver in versions:
  314                 if Version.parse(ver).gt(Version.parse(latest)):
  315                     latest = ver
  316             self.latest_version = latest
  317             self.save()
  318         return state.latest_version
  319 
  320     def get_mirrored_versions(self):
  321         if state.mirrored_versions is None:
  322             releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
  323             releases = json.loads(releases_str)['lucene']
  324             state.mirrored_versions = [ r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith('lucene-'), list(releases.keys())))) ]
  325         return state.mirrored_versions
  326 
  327     def get_mirrored_versions_to_delete(self):
  328         versions = self.get_mirrored_versions()
  329         to_keep = versions
  330         if state.release_type == 'major':
  331           to_keep = [self.release_version, self.get_latest_version()]
  332         if state.release_type == 'minor':
  333           to_keep = [self.release_version, self.get_latest_lts_version()]
  334         if state.release_type == 'bugfix':
  335           if Version.parse(state.release_version).major == Version.parse(state.get_latest_version()).major:
  336             to_keep = [self.release_version, self.get_latest_lts_version()]
  337           elif Version.parse(state.release_version).major == Version.parse(state.get_latest_lts_version()).major:
  338             to_keep = [self.get_latest_version(), self.release_version()]
  339           else:
  340             raise Exception("Release version %s must have same major version as current minor or lts release")
  341         return [ver for ver in versions if ver not in to_keep]
  342 
  343     def get_master_version(self):
  344         v = Version.parse(self.get_latest_version())
  345         return "%s.%s.%s" % (v.major + 1, 0, 0)
  346 
  347     def get_latest_lts_version(self):
  348         versions = self.get_mirrored_versions()
  349         latest = self.get_latest_version()
  350         lts_prefix = "%s." % (Version.parse(latest).major - 1)
  351         lts_versions = list(filter(lambda x: x.startswith(lts_prefix), versions))
  352         latest_lts = lts_versions[0]
  353         for ver in lts_versions:
  354             if Version.parse(ver).gt(Version.parse(latest_lts)):
  355                 latest_lts = ver
  356         return latest_lts
  357 
  358     def validate_release_version(self, branch_type, branch, release_version):
  359         ver = Version.parse(release_version)
  360         # print("release_version=%s, ver=%s" % (release_version, ver))
  361         if branch_type == BranchType.release:
  362             if not branch.startswith('branch_'):
  363                 sys.exit("Incompatible branch and branch_type")
  364             if not ver.is_bugfix_release():
  365                 sys.exit("You can only release bugfix releases from an existing release branch")
  366         elif branch_type == BranchType.stable:
  367             if not branch.startswith('branch_') and branch.endswith('x'):
  368                 sys.exit("Incompatible branch and branch_type")
  369             if not ver.is_minor_release():
  370                 sys.exit("You can only release minor releases from an existing stable branch")
  371         elif branch_type == BranchType.unstable:
  372             if not branch == 'master':
  373                 sys.exit("Incompatible branch and branch_type")
  374             if not ver.is_major_release():
  375                 sys.exit("You can only release a new major version from master branch")
  376         if not getScriptVersion() == release_version:
  377             print("WARNING: Expected release version %s when on branch %s, but got %s" % (
  378                 getScriptVersion(), branch, release_version))
  379 
  380     def get_base_branch_name(self):
  381         v = Version.parse(self.release_version)
  382         if v.is_major_release():
  383             return 'master'
  384         elif v.is_minor_release():
  385             return self.get_stable_branch_name()
  386         elif v.major == Version.parse(self.get_latest_version()).major:
  387             return self.get_minor_branch_name()
  388         else:
  389             return self.release_branch
  390 
  391     def clear_rc(self):
  392         if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):
  393             maybe_remove_rc_from_svn()
  394             dict = {}
  395             for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):
  396                 for t in g.get_todos():
  397                     t.clear()
  398             print("Cleared RC TODO state")
  399             try:
  400                 shutil.rmtree(self.get_rc_folder())
  401                 print("Cleared folder %s" % self.get_rc_folder())
  402             except Exception as e:
  403                 print("WARN: Failed to clear %s, please do it manually with higher privileges" % self.get_rc_folder())
  404             self.save()
  405 
  406     def new_rc(self):
  407         if ask_yes_no("Are you sure? This will abort current RC"):
  408             maybe_remove_rc_from_svn()
  409             dict = {}
  410             for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):
  411                 for t in g.get_todos():
  412                     if t.applies(self.release_type):
  413                         dict[t.id] = copy.deepcopy(t.state)
  414                         t.clear()
  415             self.previous_rcs["RC%d" % self.rc_number] = dict
  416             self.rc_number += 1
  417             self.save()
  418 
  419     def to_dict(self):
  420         tmp_todos = {}
  421         for todo_id in self.todos:
  422             t = self.todos[todo_id]
  423             tmp_todos[todo_id] = copy.deepcopy(t.state)
  424         dict = {
  425             'script_version': self.script_version,
  426             'release_version': self.release_version,
  427             'start_date': self.start_date,
  428             'rc_number': self.rc_number,
  429             'script_branch': self.script_branch,
  430             'todos': tmp_todos,
  431             'previous_rcs': self.previous_rcs
  432         }
  433         if self.latest_version:
  434             dict['latest_version'] = self.latest_version
  435         return dict
  436 
  437     def restore_from_dict(self, dict):
  438         self.script_version = dict['script_version']
  439         assert dict['release_version'] == self.release_version
  440         if 'start_date' in dict:
  441             self.start_date = dict['start_date']
  442         if 'latest_version' in dict:
  443             self.latest_version = dict['latest_version']
  444         else:
  445             self.latest_version = None
  446         self.rc_number = dict['rc_number']
  447         self.script_branch = dict['script_branch']
  448         self.previous_rcs = copy.deepcopy(dict['previous_rcs'])
  449         for todo_id in dict['todos']:
  450             if todo_id in self.todos:
  451                 t = self.todos[todo_id]
  452                 for k in dict['todos'][todo_id]:
  453                     t.state[k] = dict['todos'][todo_id][k]
  454             else:
  455                 print("Warning: Could not restore state for %s, Todo definition not found" % todo_id)
  456 
  457     def load(self):
  458         if os.path.exists(os.path.join(self.config_path, self.release_version, 'state.yaml')):
  459             state_file = os.path.join(self.config_path, self.release_version, 'state.yaml')
  460             with open(state_file, 'r') as fp:
  461                 try:
  462                     dict = yaml.load(fp, Loader=yaml.Loader)
  463                     self.restore_from_dict(dict)
  464                     print("Loaded state from %s" % state_file)
  465                 except Exception as e:
  466                     print("Failed to load state from %s: %s" % (state_file, e))
  467 
  468     def save(self):
  469         print("Saving")
  470         if not os.path.exists(os.path.join(self.config_path, self.release_version)):
  471             print("Creating folder %s" % os.path.join(self.config_path, self.release_version))
  472             os.makedirs(os.path.join(self.config_path, self.release_version))
  473 
  474         with open(os.path.join(self.config_path, self.release_version, 'state.yaml'), 'w') as fp:
  475             yaml.dump(self.to_dict(), fp, sort_keys=False, default_flow_style=False)
  476 
  477     def clear(self):
  478         self.previous_rcs = {}
  479         self.rc_number = 1
  480         for t_id in self.todos:
  481             t = self.todos[t_id]
  482             t.state = {}
  483         self.save()
  484 
  485     def get_rc_number(self):
  486         return self.rc_number
  487 
  488     def get_current_git_rev(self):
  489         try:
  490             return run("git rev-parse HEAD", cwd=self.get_git_checkout_folder()).strip()
  491         except:
  492             return "<git-rev>"
  493 
  494     def get_group_by_id(self, id):
  495         lst = list(filter(lambda x: x.id == id, self.todo_groups))
  496         if len(lst) == 1:
  497             return lst[0]
  498         else:
  499             return None
  500 
  501     def get_todo_by_id(self, id):
  502         lst = list(filter(lambda x: x.id == id, self.todos.values()))
  503         if len(lst) == 1:
  504             return lst[0]
  505         else:
  506             return None
  507 
  508     def get_todo_state_by_id(self, id):
  509         lst = list(filter(lambda x: x.id == id, self.todos.values()))
  510         if len(lst) == 1:
  511             return lst[0].state
  512         else:
  513             return {}
  514 
  515     def get_release_folder(self):
  516         folder = os.path.join(self.config_path, self.release_version)
  517         if not os.path.exists(folder):
  518             print("Creating folder %s" % folder)
  519             os.makedirs(folder)
  520         return folder
  521 
  522     def get_rc_folder(self):
  523         folder = os.path.join(self.get_release_folder(), "RC%d" % self.rc_number)
  524         if not os.path.exists(folder):
  525             print("Creating folder %s" % folder)
  526             os.makedirs(folder)
  527         return folder
  528 
  529     def get_dist_folder(self):
  530         folder = os.path.join(self.get_rc_folder(), "dist")
  531         return folder
  532 
  533     def get_git_checkout_folder(self):
  534         folder = os.path.join(self.get_release_folder(), "lucene-solr")
  535         return folder
  536 
  537     def get_minor_branch_name(self):
  538         latest = state.get_latest_version()
  539         if latest is not None:
  540           v = Version.parse(latest)
  541           return "branch_%s_%s" % (v.major, v.minor)
  542         else:
  543             raise Exception("Cannot find latest version")
  544 
  545     def get_stable_branch_name(self):
  546         v = Version.parse(self.get_latest_version())
  547         return "branch_%sx" % v.major
  548 
  549     def get_next_version(self):
  550         if self.release_type == 'major':
  551             return "%s.0" % (self.release_version_major + 1)
  552         if self.release_type == 'minor':
  553             return "%s.%s" % (self.release_version_major, self.release_version_minor + 1)
  554         if self.release_type == 'bugfix':
  555             return "%s.%s.%s" % (self.release_version_major, self.release_version_minor, self.release_version_bugfix + 1)
  556 
  557     def get_java_home(self):
  558         return self.get_java_home_for_version(self.release_version)
  559 
  560     def get_java_home_for_version(self, version):
  561         v = Version.parse(version)
  562         java_ver = java_versions[v.major]
  563         java_home_var = "JAVA%s_HOME" % java_ver
  564         if java_home_var in os.environ:
  565             return os.environ.get(java_home_var)
  566         else:
  567             raise Exception("Script needs environment variable %s" % java_home_var )
  568 
  569     def get_java_cmd_for_version(self, version):
  570         return os.path.join(self.get_java_home_for_version(version), "bin", "java")
  571 
  572     def get_java_cmd(self):
  573         return os.path.join(self.get_java_home(), "bin", "java")
  574 
  575     def get_todo_states(self):
  576         states = {}
  577         if self.todos:
  578             for todo_id in self.todos:
  579                 t = self.todos[todo_id]
  580                 states[todo_id] = copy.deepcopy(t.state)
  581         return states
  582 
  583     def init_todos(self, groups):
  584         self.todo_groups = groups
  585         self.todos = {}
  586         for g in self.todo_groups:
  587             for t in g.get_todos():
  588                 self.todos[t.id] = t
  589 
  590 
  591 class TodoGroup(SecretYamlObject):
  592     yaml_tag = u'!TodoGroup'
  593     hidden_fields = []
  594     def __init__(self, id, title, description, todos, is_in_rc_loop=None, depends=None):
  595         self.id = id
  596         self.title = title
  597         self.description = description
  598         self.depends = depends
  599         self.is_in_rc_loop = is_in_rc_loop
  600         self.todos = todos
  601 
  602     @classmethod
  603     def from_yaml(cls, loader, node):
  604         fields = loader.construct_mapping(node, deep = True)
  605         return TodoGroup(**fields)
  606 
  607     def num_done(self):
  608         return sum(1 for x in self.todos if x.is_done() > 0)
  609 
  610     def num_applies(self):
  611         count = sum(1 for x in self.todos if x.applies(state.release_type))
  612         # print("num_applies=%s" % count)
  613         return count
  614 
  615     def is_done(self):
  616         # print("Done=%s, applies=%s" % (self.num_done(), self.num_applies()))
  617         return self.num_done() >= self.num_applies()
  618 
  619     def get_title(self):
  620         # print("get_title: %s" % self.is_done())
  621         prefix = ""
  622         if self.is_done():
  623             prefix = "✓ "
  624         return "%s%s (%d/%d)" % (prefix, self.title, self.num_done(), self.num_applies())
  625 
  626     def get_submenu(self):
  627         menu = UpdatableConsoleMenu(title=self.title, subtitle=self.get_subtitle, prologue_text=self.get_description(),
  628                            screen=MyScreen())
  629         menu.exit_item = CustomExitItem("Return")
  630         for todo in self.get_todos():
  631             if todo.applies(state.release_type):
  632                 menu.append_item(todo.get_menu_item())
  633         return menu
  634 
  635     def get_menu_item(self):
  636         item = UpdatableSubmenuItem(self.get_title, self.get_submenu())
  637         return item
  638 
  639     def get_todos(self):
  640         return self.todos
  641 
  642     def in_rc_loop(self):
  643         return self.is_in_rc_loop is True
  644 
  645     def get_description(self):
  646         desc = self.description
  647         if desc:
  648             return expand_jinja(desc)
  649         else:
  650             return None
  651 
  652     def get_subtitle(self):
  653         if self.depends:
  654             ret_str = ""
  655             for dep in ensure_list(self.depends):
  656                 g = state.get_group_by_id(dep)
  657                 if not g:
  658                     g = state.get_todo_by_id(dep)
  659                 if g and not g.is_done():
  660                     ret_str += "NOTE: Please first complete '%s'\n" % g.title
  661                     return ret_str.strip()
  662         return None
  663 
  664 
  665 class Todo(SecretYamlObject):
  666     yaml_tag = u'!Todo'
  667     hidden_fields = ['state']
  668     def __init__(self, id, title, description=None, post_description=None, done=None, types=None, links=None,
  669                  commands=None, user_input=None, depends=None, vars=None, asciidoc=None, persist_vars=None,
  670                  function=None):
  671         self.id = id
  672         self.title = title
  673         self.description = description
  674         self.asciidoc = asciidoc
  675         self.types = types
  676         self.depends = depends
  677         self.vars = vars
  678         self.persist_vars = persist_vars
  679         self.function = function
  680         self.user_input = user_input
  681         self.commands = commands
  682         self.post_description = post_description
  683         self.links = links
  684         self.state = {}
  685 
  686         self.set_done(done)
  687         if self.types:
  688             self.types = ensure_list(self.types)
  689             for t in self.types:
  690                 if not t in ['minor', 'major', 'bugfix']:
  691                     sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
  692         if commands:
  693             self.commands.todo_id = self.id
  694             for c in commands.commands:
  695                 c.todo_id = self.id
  696 
  697     @classmethod
  698     def from_yaml(cls, loader, node):
  699         fields = loader.construct_mapping(node, deep = True)
  700         return Todo(**fields)
  701 
  702     def get_vars(self):
  703         myvars = {}
  704         if self.vars:
  705             for k in self.vars:
  706                 val = self.vars[k]
  707                 if callable(val):
  708                     myvars[k] = expand_jinja(val(), vars=myvars)
  709                 else:
  710                     myvars[k] = expand_jinja(val, vars=myvars)
  711         return myvars
  712 
  713     def set_done(self, is_done):
  714         if is_done:
  715             self.state['done_date'] = unix_time_millis(datetime.utcnow())
  716             if self.persist_vars:
  717                 for k in self.persist_vars:
  718                     self.state[k] = self.get_vars()[k]
  719         else:
  720             self.state.clear()
  721         self.state['done'] = is_done
  722 
  723     def applies(self, type):
  724         if self.types:
  725             return type in self.types
  726         return True
  727 
  728     def is_done(self):
  729         return 'done' in self.state and self.state['done'] is True
  730 
  731     def get_title(self):
  732         prefix = ""
  733         if self.is_done():
  734             prefix = "✓ "
  735         return expand_jinja("%s%s" % (prefix, self.title), self.get_vars_and_state())
  736 
  737     def display_and_confirm(self):
  738         try:
  739             if self.depends:
  740                 ret_str = ""
  741                 for dep in ensure_list(self.depends):
  742                     g = state.get_group_by_id(dep)
  743                     if not g:
  744                         g = state.get_todo_by_id(dep)
  745                     if not g.is_done():
  746                         print("This step depends on '%s'. Please complete that first\n" % g.title)
  747                         return
  748             desc = self.get_description()
  749             if desc:
  750                 print("%s" % desc)
  751             try:
  752                 if self.function and not self.is_done():
  753                     if not eval(self.function)(self):
  754                         return
  755             except Exception as e:
  756                 print("Function call to %s for todo %s failed: %s" % (self.function, self.id, e))
  757                 raise e
  758             if self.user_input and not self.is_done():
  759                 ui_list = ensure_list(self.user_input)
  760                 for ui in ui_list:
  761                     ui.run(self.state)
  762                 print()
  763             if self.links:
  764                 print("\nLinks:\n")
  765                 for link in self.links:
  766                     print("- %s" % expand_jinja(link, self.get_vars_and_state()))
  767                 print()
  768             cmds = self.get_commands()
  769             if cmds:
  770                 if not self.is_done():
  771                     if not cmds.logs_prefix:
  772                         cmds.logs_prefix = self.id
  773                     cmds.run()
  774                 else:
  775                     print("This step is already completed. You have to first set it to 'not completed' in order to execute commands again.")
  776                 print()
  777             if self.post_description:
  778                 print("%s" % self.get_post_description())
  779             todostate = self.get_state()
  780             if self.is_done() and len(todostate) > 2:
  781                 print("Variables registered\n")
  782                 for k in todostate:
  783                     if k == 'done' or k == 'done_date':
  784                         continue
  785                     print("* %s = %s" % (k, todostate[k]))
  786                 print()
  787             completed = ask_yes_no("Mark task '%s' as completed?" % self.get_title())
  788             self.set_done(completed)
  789             state.save()
  790         except Exception as e:
  791             print("ERROR while executing todo %s (%s)" % (self.get_title(), e))
  792 
  793     def get_menu_item(self):
  794         return UpdatableFunctionItem(self.get_title, self.display_and_confirm)
  795 
  796     def clone(self):
  797         clone = Todo(self.id, self.title, description=self.description)
  798         clone.state = copy.deepcopy(self.state)
  799         return clone
  800 
  801     def clear(self):
  802         self.state.clear()
  803         self.set_done(False)
  804 
  805     def get_state(self):
  806         return self.state
  807 
  808     def get_description(self):
  809         desc = self.description
  810         if desc:
  811             return expand_jinja(desc, vars=self.get_vars_and_state())
  812         else:
  813             return None
  814 
  815     def get_post_description(self):
  816         if self.post_description:
  817             return expand_jinja(self.post_description, vars=self.get_vars_and_state())
  818         else:
  819             return None
  820 
  821     def get_commands(self):
  822         cmds = self.commands
  823         return cmds
  824 
  825     def get_asciidoc(self):
  826         if self.asciidoc:
  827             return expand_jinja(self.asciidoc, vars=self.get_vars_and_state())
  828         else:
  829             return None
  830 
  831     def get_vars_and_state(self):
  832         d = self.get_vars().copy()
  833         d.update(self.get_state())
  834         return d
  835 
  836 
  837 def get_release_version():
  838     v = str(input("Which version are you releasing? (x.y.z) "))
  839     try:
  840         version = Version.parse(v)
  841     except:
  842         print("Not a valid version %s" % v)
  843         return get_release_version()
  844 
  845     return str(version)
  846 
  847 
  848 def get_subtitle():
  849     applying_groups = list(filter(lambda x: x.num_applies() > 0, state.todo_groups))
  850     done_groups = sum(1 for x in applying_groups if x.is_done())
  851     return "Please complete the below checklist (Complete: %s/%s)" % (done_groups, len(applying_groups))
  852 
  853 
  854 def get_todo_menuitem_title():
  855     return "Go to checklist (RC%d)" % (state.rc_number)
  856 
  857 
  858 def get_releasing_text():
  859     return "Releasing Lucene/Solr %s RC%d" % (state.release_version, state.rc_number)
  860 
  861 
  862 def get_start_new_rc_menu_title():
  863     return "Abort RC%d and start a new RC%d" % (state.rc_number, state.rc_number + 1)
  864 
  865 
  866 def start_new_rc():
  867     state.new_rc()
  868     print("Started RC%d" % state.rc_number)
  869 
  870 
  871 def reset_state():
  872     global state
  873     if ask_yes_no("Are you sure? This will erase all current progress"):
  874         maybe_remove_rc_from_svn()
  875         shutil.rmtree(os.path.join(state.config_path, state.release_version))
  876         state.clear()
  877 
  878 
  879 def template(name, vars=None):
  880     return expand_jinja(templates[name], vars=vars)
  881 
  882 
  883 def help():
  884     print(template('help'))
  885     pause()
  886 
  887 
  888 def ensure_list(o):
  889     if o is None:
  890         return []
  891     if not isinstance(o, list):
  892         return [o]
  893     else:
  894         return o
  895 
  896 
  897 def open_file(filename):
  898     print("Opening file %s" % filename)
  899     if platform.system().startswith("Win"):
  900         run("start %s" % filename)
  901     else:
  902         run("open %s" % filename)
  903 
  904 
  905 def expand_multiline(cmd_txt, indent=0):
  906     return re.sub(r'  +', " %s\n    %s" % (Commands.cmd_continuation_char, " "*indent), cmd_txt)
  907 
  908 
  909 def unix_to_datetime(unix_stamp):
  910     return datetime.utcfromtimestamp(unix_stamp / 1000)
  911 
  912 
  913 def generate_asciidoc():
  914     base_filename = os.path.join(state.get_release_folder(),
  915                                  "lucene_solr_release_%s"
  916                                  % (state.release_version.replace("\.", "_")))
  917 
  918     filename_adoc = "%s.adoc" % base_filename
  919     filename_html = "%s.html" % base_filename
  920     fh = open(filename_adoc, "w")
  921 
  922     fh.write("= Lucene/Solr Release %s\n\n" % state.release_version)
  923     fh.write("(_Generated by releaseWizard.py v%s ALPHA at %s_)\n\n"
  924              % (getScriptVersion(), datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")))
  925     fh.write(":numbered:\n\n")
  926     fh.write("%s\n\n" % template('help'))
  927     for group in state.todo_groups:
  928         if group.num_applies() == 0:
  929             continue
  930         fh.write("== %s\n\n" % group.get_title())
  931         fh.write("%s\n\n" % group.get_description())
  932         for todo in group.get_todos():
  933             if not todo.applies(state.release_type):
  934                 continue
  935             fh.write("=== %s\n\n" % todo.get_title())
  936             if todo.is_done():
  937                 fh.write("_Completed %s_\n\n" % unix_to_datetime(todo.state['done_date']).strftime(
  938                     "%Y-%m-%d %H:%M UTC"))
  939             if todo.get_asciidoc():
  940                 fh.write("%s\n\n" % todo.get_asciidoc())
  941             else:
  942                 desc = todo.get_description()
  943                 if desc:
  944                     fh.write("%s\n\n" % desc)
  945             state_copy = copy.deepcopy(todo.state)
  946             state_copy.pop('done', None)
  947             state_copy.pop('done_date', None)
  948             if len(state_copy) > 0 or todo.user_input is not None:
  949                 fh.write(".Variables collected in this step\n")
  950                 fh.write("|===\n")
  951                 fh.write("|Variable |Value\n")
  952                 mykeys = set()
  953                 for e in ensure_list(todo.user_input):
  954                     mykeys.add(e.name)
  955                 for e in state_copy.keys():
  956                     mykeys.add(e)
  957                 for key in mykeys:
  958                     val = "(not set)"
  959                     if key in state_copy:
  960                         val = state_copy[key]
  961                     fh.write("\n|%s\n|%s\n" % (key, val))
  962                 fh.write("|===\n\n")
  963             cmds = todo.get_commands()
  964             if cmds:
  965                 if cmds.commands_text:
  966                     fh.write("%s\n\n" % cmds.get_commands_text())
  967                 fh.write("[source,sh]\n----\n")
  968                 if cmds.env:
  969                     for key in cmds.env:
  970                         val = cmds.env[key]
  971                         if is_windows():
  972                             fh.write("SET %s=%s\n" % (key, val))
  973                         else:
  974                             fh.write("export %s=%s\n" % (key, val))
  975                 fh.write(abbreviate_homedir("cd %s\n" % cmds.get_root_folder()))
  976                 cmds2 = ensure_list(cmds.commands)
  977                 for c in cmds2:
  978                     for line in c.display_cmd():
  979                         fh.write("%s\n" % line)
  980                 fh.write("----\n\n")
  981             if todo.post_description and not todo.get_asciidoc():
  982                 fh.write("\n%s\n\n" % todo.get_post_description())
  983             if todo.links:
  984                 fh.write("Links:\n\n")
  985                 for l in todo.links:
  986                     fh.write("* %s\n" % expand_jinja(l))
  987                 fh.write("\n")
  988 
  989     fh.close()
  990     print("Wrote file %s" % os.path.join(state.get_release_folder(), filename_adoc))
  991     print("Running command 'asciidoctor %s'" % filename_adoc)
  992     run_follow("asciidoctor %s" % filename_adoc)
  993     if os.path.exists(filename_html):
  994         open_file(filename_html)
  995     else:
  996         print("Failed generating HTML version, please install asciidoctor")
  997     pause()
  998 
  999 
 1000 def load_rc():
 1001     lucenerc = os.path.expanduser("~/.lucenerc")
 1002     try:
 1003         with open(lucenerc, 'r') as fp:
 1004             return json.load(fp)
 1005     except:
 1006         return None
 1007 
 1008 
 1009 def store_rc(release_root, release_version=None):
 1010     lucenerc = os.path.expanduser("~/.lucenerc")
 1011     dict = {}
 1012     dict['root'] = release_root
 1013     if release_version:
 1014         dict['release_version'] = release_version
 1015     with open(lucenerc, "w") as fp:
 1016         json.dump(dict, fp, indent=2)
 1017 
 1018 
 1019 def release_other_version():
 1020     if not state.is_released():
 1021         maybe_remove_rc_from_svn()
 1022     store_rc(state.config_path, None)
 1023     print("Please restart the wizard")
 1024     sys.exit(0)
 1025 
 1026 def file_to_string(filename):
 1027     with open(filename, encoding='utf8') as f:
 1028         return f.read().strip()
 1029 
 1030 def download_keys():
 1031     download('KEYS', "https://archive.apache.org/dist/lucene/KEYS", state.config_path)
 1032 
 1033 def keys_downloaded():
 1034     return os.path.exists(os.path.join(state.config_path, "KEYS"))
 1035 
 1036 
 1037 def dump_yaml():
 1038     file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")
 1039     yaml.add_representer(str, str_presenter)
 1040     yaml.Dumper.ignore_aliases = lambda *args : True
 1041     dump_obj = {'templates': templates,
 1042                 'groups': state.todo_groups}
 1043     yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)
 1044 
 1045 
 1046 def parse_config():
 1047     description = 'Script to guide a RM through the whole release process'
 1048     parser = argparse.ArgumentParser(description=description, epilog="Go push that release!",
 1049                                      formatter_class=argparse.RawDescriptionHelpFormatter)
 1050     parser.add_argument('--dry-run', dest='dry', action='store_true', default=False,
 1051                         help='Do not execute any commands, but echo them instead. Display extra debug info')
 1052     parser.add_argument('--init', action='store_true', default=False,
 1053                         help='Re-initialize root and version')
 1054     config = parser.parse_args()
 1055 
 1056     return config
 1057 
 1058 
 1059 def load(urlString, encoding="utf-8"):
 1060     try:
 1061         content = urllib.request.urlopen(urlString).read().decode(encoding)
 1062     except Exception as e:
 1063         print('Retrying download of url %s after exception: %s' % (urlString, e))
 1064         content = urllib.request.urlopen(urlString).read().decode(encoding)
 1065     return content
 1066 
 1067 
 1068 def configure_pgp(gpg_todo):
 1069     print("Based on your Apache ID we'll lookup your key online\n"
 1070           "and through this complete the 'gpg' prerequisite task.\n")
 1071     gpg_state = gpg_todo.get_state()
 1072     id = str(input("Please enter your Apache id: (ENTER=skip) "))
 1073     if id.strip() == '':
 1074         return False
 1075     all_keys = load('https://home.apache.org/keys/group/lucene.asc')
 1076     lines = all_keys.splitlines()
 1077     keyid_linenum = None
 1078     for idx, line in enumerate(lines):
 1079         if line == 'ASF ID: %s' % id:
 1080             keyid_linenum = idx+1
 1081             break
 1082     if keyid_linenum:
 1083         keyid_line = lines[keyid_linenum]
 1084         assert keyid_line.startswith('LDAP PGP key: ')
 1085         gpg_id = keyid_line[14:].replace(" ", "")[-8:]
 1086         print("Found gpg key id %s on file at Apache (https://home.apache.org/keys/group/lucene.asc)" % gpg_id)
 1087     else:
 1088         print(textwrap.dedent("""\
 1089             Could not find your GPG key from Apache servers.
 1090             Please make sure you have registered your key ID in
 1091             id.apache.org, see links for more info."""))
 1092         gpg_id = str(input("Enter your key ID manually, 8 last characters (ENTER=skip): "))
 1093         if gpg_id.strip() == '':
 1094             return False
 1095         elif len(gpg_id) != 8:
 1096             print("gpg id must be the last 8 characters of your key id")
 1097         gpg_id = gpg_id.upper()
 1098     try:
 1099         res = run("gpg --list-secret-keys %s" % gpg_id)
 1100         print("Found key %s on your private gpg keychain" % gpg_id)
 1101         # Check rsa and key length >= 4096
 1102         match = re.search(r'^sec +((rsa|dsa)(\d{4})) ', res)
 1103         type = "(unknown)"
 1104         length = -1
 1105         if match:
 1106             type = match.group(2)
 1107             length = int(match.group(3))
 1108         else:
 1109             match = re.search(r'^sec +((\d{4})([DR])/.*?) ', res)
 1110             if match:
 1111                 type = 'rsa' if match.group(3) == 'R' else 'dsa'
 1112                 length = int(match.group(2))
 1113             else:
 1114                 print("Could not determine type and key size for your key")
 1115                 print("%s" % res)
 1116                 if not ask_yes_no("Is your key of type RSA and size >= 2048 (ideally 4096)? "):
 1117                     print("Sorry, please generate a new key, add to KEYS and register with id.apache.org")
 1118                     return False
 1119         if not type == 'rsa':
 1120             print("We strongly recommend RSA type key, your is '%s'. Consider generating a new key." % type.upper())
 1121         if length < 2048:
 1122             print("Your key has key length of %s. Cannot use < 2048, please generate a new key before doing the release" % length)
 1123             return False
 1124         if length < 4096:
 1125             print("Your key length is < 4096, Please generate a stronger key.")
 1126             print("Alternatively, follow instructions in http://www.apache.org/dev/release-signing.html#note")
 1127             if not ask_yes_no("Have you configured your gpg to avoid SHA-1?"):
 1128                 print("Please either generate a strong key or reconfigure your client")
 1129                 return False
 1130         print("Validated that your key is of type RSA and has a length >= 2048 (%s)" % length)
 1131     except:
 1132         print(textwrap.dedent("""\
 1133             Key not found on your private gpg keychain. In order to sign the release you'll
 1134             need to fix this, then try again"""))
 1135         return False
 1136     try:
 1137         lines = run("gpg --check-signatures %s" % gpg_id).splitlines()
 1138         sigs = 0
 1139         apache_sigs = 0
 1140         for line in lines:
 1141             if line.startswith("sig") and not gpg_id in line:
 1142                 sigs += 1
 1143                 if '@apache.org' in line:
 1144                     apache_sigs += 1
 1145         print("Your key has %s signatures, of which %s are by committers (@apache.org address)" % (sigs, apache_sigs))
 1146         if apache_sigs < 1:
 1147             print(textwrap.dedent("""\
 1148                 Your key is not signed by any other committer. 
 1149                 Please review http://www.apache.org/dev/openpgp.html#apache-wot
 1150                 and make sure to get your key signed until next time.
 1151                 You may want to run 'gpg --refresh-keys' to refresh your keychain."""))
 1152         uses_apacheid = is_code_signing_key = False
 1153         for line in lines:
 1154             if line.startswith("uid") and "%s@apache" % id in line:
 1155                 uses_apacheid = True
 1156                 if 'CODE SIGNING KEY' in line.upper():
 1157                     is_code_signing_key = True
 1158         if not uses_apacheid:
 1159             print("WARNING: Your key should use your apache-id email address, see http://www.apache.org/dev/release-signing.html#user-id")
 1160         if not is_code_signing_key:
 1161             print("WARNING: You code signing key should be labeled 'CODE SIGNING KEY', see http://www.apache.org/dev/release-signing.html#key-comment")
 1162     except Exception as e:
 1163         print("Could not check signatures of your key: %s" % e)
 1164 
 1165     download_keys()
 1166     keys_text = file_to_string(os.path.join(state.config_path, "KEYS"))
 1167     if gpg_id in keys_text or "%s %s" % (gpg_id[:4], gpg_id[-4:]) in keys_text:
 1168         print("Found your key ID in official KEYS file. KEYS file is not cached locally.")
 1169     else:
 1170         print(textwrap.dedent("""\
 1171             Could not find your key ID in official KEYS file.
 1172             Please make sure it is added to https://dist.apache.org/repos/dist/release/lucene/KEYS
 1173             and committed to svn. Then re-try this initialization"""))
 1174         if not ask_yes_no("Do you want to continue without fixing KEYS file? (not recommended) "):
 1175             return False
 1176 
 1177     gpg_state['apache_id'] = id
 1178     gpg_state['gpg_key'] = gpg_id
 1179     return True
 1180 
 1181 
 1182 def pause(fun=None):
 1183     if fun:
 1184         fun()
 1185     input("\nPress ENTER to continue...")
 1186 
 1187 
 1188 # Custom classes for ConsoleMenu, to make menu texts dynamic
 1189 # Needed until https://github.com/aegirhall/console-menu/pull/25 is released
 1190 # See https://pypi.org/project/console-menu/ for other docs
 1191 
 1192 class UpdatableConsoleMenu(ConsoleMenu):
 1193 
 1194     def __repr__(self):
 1195         return "%s: %s. %d items" % (self.get_title(), self.get_subtitle(), len(self.items))
 1196 
 1197     def draw(self):
 1198         """
 1199         Refreshes the screen and redraws the menu. Should be called whenever something changes that needs to be redrawn.
 1200         """
 1201         self.screen.printf(self.formatter.format(title=self.get_title(), subtitle=self.get_subtitle(), items=self.items,
 1202                                                  prologue_text=self.get_prologue_text(), epilogue_text=self.get_epilogue_text()))
 1203 
 1204     # Getters to get text in case method reference
 1205     def get_title(self):
 1206         return self.title() if callable(self.title) else self.title
 1207 
 1208     def get_subtitle(self):
 1209         return self.subtitle() if callable(self.subtitle) else self.subtitle
 1210 
 1211     def get_prologue_text(self):
 1212         return self.prologue_text() if callable(self.prologue_text) else self.prologue_text
 1213 
 1214     def get_epilogue_text(self):
 1215         return self.epilogue_text() if callable(self.epilogue_text) else self.epilogue_text
 1216 
 1217 
 1218 class UpdatableSubmenuItem(SubmenuItem):
 1219     def __init__(self, text, submenu, menu=None, should_exit=False):
 1220         """
 1221         :ivar ConsoleMenu self.submenu: The submenu to be opened when this item is selected
 1222         """
 1223         super(SubmenuItem, self).__init__(text=text, menu=menu, should_exit=should_exit)
 1224 
 1225         self.submenu = submenu
 1226         if menu:
 1227             self.get_submenu().parent = menu
 1228 
 1229     def show(self, index):
 1230         return "%2d - %s" % (index + 1, self.get_text())
 1231 
 1232     # Getters to get text in case method reference
 1233     def get_text(self):
 1234         return self.text() if callable(self.text) else self.text
 1235 
 1236     def set_menu(self, menu):
 1237         """
 1238         Sets the menu of this item.
 1239         Should be used instead of directly accessing the menu attribute for this class.
 1240 
 1241         :param ConsoleMenu menu: the menu
 1242         """
 1243         self.menu = menu
 1244         self.get_submenu().parent = menu
 1245 
 1246     def action(self):
 1247         """
 1248         This class overrides this method
 1249         """
 1250         self.get_submenu().start()
 1251 
 1252     def clean_up(self):
 1253         """
 1254         This class overrides this method
 1255         """
 1256         self.get_submenu().join()
 1257         self.menu.clear_screen()
 1258         self.menu.resume()
 1259 
 1260     def get_return(self):
 1261         """
 1262         :return: The returned value in the submenu
 1263         """
 1264         return self.get_submenu().returned_value
 1265 
 1266     def get_submenu(self):
 1267         """
 1268         We unwrap the submenu variable in case it is a reference to a method that returns a submenu
 1269         """
 1270         return self.submenu if not callable(self.submenu) else self.submenu()
 1271 
 1272 
 1273 class UpdatableFunctionItem(FunctionItem):
 1274     def show(self, index):
 1275         return "%2d - %s" % (index + 1, self.get_text())
 1276 
 1277     # Getters to get text in case method reference
 1278     def get_text(self):
 1279         return self.text() if callable(self.text) else self.text
 1280 
 1281 
 1282 class MyScreen(Screen):
 1283     def clear(self):
 1284         return
 1285 
 1286 
 1287 class CustomExitItem(ExitItem):
 1288     def show(self, index):
 1289         return super(ExitItem, self).show(index)
 1290 
 1291     def get_return(self):
 1292         return ""
 1293 
 1294 
 1295 def main():
 1296     global state
 1297     global dry_run
 1298     global templates
 1299 
 1300     print("Lucene/Solr releaseWizard v%s" % getScriptVersion())
 1301     c = parse_config()
 1302 
 1303     if c.dry:
 1304         print("Entering dry-run mode where all commands will be echoed instead of executed")
 1305         dry_run = True
 1306 
 1307     release_root = os.path.expanduser("~/.lucene-releases")
 1308     if not load_rc() or c.init:
 1309         print("Initializing")
 1310         dir_ok = False
 1311         root = str(input("Choose root folder: [~/.lucene-releases] "))
 1312         if os.path.exists(root) and (not os.path.isdir(root) or not os.access(root, os.W_OK)):
 1313             sys.exit("Root %s exists but is not a directory or is not writable" % root)
 1314         if not root == '':
 1315             if root.startswith("~/"):
 1316                 release_root = os.path.expanduser(root)
 1317             else:
 1318                 release_root = os.path.abspath(root)
 1319         if not os.path.exists(release_root):
 1320             try:
 1321                 print("Creating release root %s" % release_root)
 1322                 os.makedirs(release_root)
 1323             except Exception as e:
 1324                 sys.exit("Error while creating %s: %s" % (release_root, e))
 1325         release_version = get_release_version()
 1326     else:
 1327         conf = load_rc()
 1328         release_root = conf['root']
 1329         if 'release_version' in conf:
 1330             release_version = conf['release_version']
 1331         else:
 1332             release_version = get_release_version()
 1333     store_rc(release_root, release_version)
 1334 
 1335 
 1336     check_prerequisites()
 1337 
 1338     try:
 1339         y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml"), "r"), Loader=yaml.Loader)
 1340         templates = y.get('templates')
 1341         todo_list = y.get('groups')
 1342         state = ReleaseState(release_root, release_version, getScriptVersion())
 1343         state.init_todos(bootstrap_todos(todo_list))
 1344         state.load()
 1345     except Exception as e:
 1346         sys.exit("Failed initializing. %s" % e)
 1347 
 1348     state.save()
 1349 
 1350     # Smoketester requires JAVA_HOME to point to JAVA8 and JAVA11_HOME to point ot Java11
 1351     os.environ['JAVA_HOME'] = state.get_java_home()
 1352     os.environ['JAVACMD'] = state.get_java_cmd()
 1353 
 1354     global tlp_news_draft
 1355     global lucene_news_draft
 1356     global solr_news_draft
 1357     global lucene_highlights_file
 1358     global solr_highlights_file
 1359     global website_folder
 1360     global tlp_news_file
 1361     global lucene_news_file
 1362     global solr_news_file
 1363     lucene_highlights_file = os.path.join(state.get_release_folder(), 'lucene_highlights.txt')
 1364     solr_highlights_file = os.path.join(state.get_release_folder(), 'solr_highlights.txt')
 1365     tlp_news_draft = os.path.join(state.get_release_folder(), 'tlp_news.md')
 1366     lucene_news_draft = os.path.join(state.get_release_folder(), 'lucene_news.md')
 1367     solr_news_draft = os.path.join(state.get_release_folder(), 'solr_news.md')
 1368     website_folder = os.path.join(state.get_release_folder(), 'website-source')
 1369     tlp_news_file = os.path.join(website_folder, 'content', 'mainnews.mdtext')
 1370     lucene_news_file = os.path.join(website_folder, 'content', 'core', 'corenews.mdtext')
 1371     solr_news_file = os.path.join(website_folder, 'content', 'solr', 'news.mdtext')
 1372 
 1373     main_menu = UpdatableConsoleMenu(title="Lucene/Solr ReleaseWizard",
 1374                             subtitle=get_releasing_text,
 1375                             prologue_text="Welcome to the release wizard. From here you can manage the process including creating new RCs. "
 1376                                           "All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.",
 1377                             epilogue_text="® 2019 The Lucene/Solr project. Licensed under the Apache License 2.0\nScript version v%s ALPHA)" % getScriptVersion(),
 1378                             screen=MyScreen())
 1379 
 1380     todo_menu = UpdatableConsoleMenu(title=get_releasing_text,
 1381                             subtitle=get_subtitle,
 1382                             prologue_text=None,
 1383                             epilogue_text=None,
 1384                             screen=MyScreen())
 1385     todo_menu.exit_item = CustomExitItem("Return")
 1386 
 1387     for todo_group in state.todo_groups:
 1388         if todo_group.num_applies() >= 0:
 1389             menu_item = todo_group.get_menu_item()
 1390             menu_item.set_menu(todo_menu)
 1391             todo_menu.append_item(menu_item)
 1392 
 1393     main_menu.append_item(UpdatableSubmenuItem(get_todo_menuitem_title, todo_menu, menu=main_menu))
 1394     main_menu.append_item(UpdatableFunctionItem(get_start_new_rc_menu_title, start_new_rc))
 1395     main_menu.append_item(UpdatableFunctionItem('Clear and restart current RC', state.clear_rc))
 1396     main_menu.append_item(UpdatableFunctionItem("Clear all state, restart the %s release" % state.release_version, reset_state))
 1397     main_menu.append_item(UpdatableFunctionItem('Start release for a different version', release_other_version))
 1398     main_menu.append_item(UpdatableFunctionItem('Generate Asciidoc guide for this release', generate_asciidoc))
 1399     # main_menu.append_item(UpdatableFunctionItem('Dump YAML', dump_yaml))
 1400     main_menu.append_item(UpdatableFunctionItem('Help', help))
 1401 
 1402     main_menu.show()
 1403 
 1404 
 1405 sys.path.append(os.path.dirname(__file__))
 1406 current_git_root = os.path.abspath(
 1407     os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir, os.path.pardir))
 1408 
 1409 dry_run = False
 1410 
 1411 major_minor = ['major', 'minor']
 1412 script_path = os.path.dirname(os.path.realpath(__file__))
 1413 os.chdir(script_path)
 1414 
 1415 
 1416 def git_checkout_folder():
 1417     return state.get_git_checkout_folder()
 1418 
 1419 
 1420 def tail_file(file, lines):
 1421     bufsize = 8192
 1422     fsize = os.stat(file).st_size
 1423     with open(file) as f:
 1424         if bufsize >= fsize:
 1425             bufsize = fsize
 1426         idx = 0
 1427         while True:
 1428             idx += 1
 1429             seek_pos = fsize - bufsize * idx
 1430             if seek_pos < 0:
 1431                 seek_pos = 0
 1432             f.seek(seek_pos)
 1433             data = []
 1434             data.extend(f.readlines())
 1435             if len(data) >= lines or f.tell() == 0 or seek_pos == 0:
 1436                 if not seek_pos == 0:
 1437                     print("Tailing last %d lines of file %s" % (lines, file))
 1438                 print(''.join(data[-lines:]))
 1439                 break
 1440 
 1441 
 1442 def run_with_log_tail(command, cwd, logfile=None, tail_lines=10, tee=False, live=False, shell=None):
 1443     fh = sys.stdout
 1444     if logfile:
 1445         logdir = os.path.dirname(logfile)
 1446         if not os.path.exists(logdir):
 1447             os.makedirs(logdir)
 1448         fh = open(logfile, 'w')
 1449     rc = run_follow(command, cwd, fh=fh, tee=tee, live=live, shell=shell)
 1450     if logfile:
 1451         fh.close()
 1452         if not tee and tail_lines and tail_lines > 0:
 1453             tail_file(logfile, tail_lines)
 1454     return rc
 1455 
 1456 
 1457 def ask_yes_no(text):
 1458     answer = None
 1459     while answer not in ['y', 'n']:
 1460         answer = str(input("\nQ: %s (y/n): " % text))
 1461     print("\n")
 1462     return answer == 'y'
 1463 
 1464 
 1465 def abbreviate_line(line, width):
 1466     line = line.rstrip()
 1467     if len(line) > width:
 1468         line = "%s.....%s" % (line[:(width / 2 - 5)], line[-(width / 2):])
 1469     else:
 1470         line = "%s%s" % (line, " " * (width - len(line) + 2))
 1471     return line
 1472 
 1473 
 1474 def print_line_cr(line, linenum, stdout=True, tee=False):
 1475     if not tee:
 1476         if not stdout:
 1477             print("[line %s] %s" % (linenum, abbreviate_line(line, 80)), end='\r')
 1478     else:
 1479         if line.endswith("\r"):
 1480             print(line.rstrip(), end='\r')
 1481         else:
 1482             print(line.rstrip())
 1483 
 1484 
 1485 def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=None):
 1486     doShell = '&&' in command or '&' in command or shell is not None
 1487     if not doShell and not isinstance(command, list):
 1488         command = shlex.split(command)
 1489     process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd,
 1490                                universal_newlines=True, bufsize=0, close_fds=True, shell=doShell)
 1491     lines_written = 0
 1492 
 1493     fl = fcntl.fcntl(process.stdout, fcntl.F_GETFL)
 1494     fcntl.fcntl(process.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK)
 1495 
 1496     flerr = fcntl.fcntl(process.stderr, fcntl.F_GETFL)
 1497     fcntl.fcntl(process.stderr, fcntl.F_SETFL, flerr | os.O_NONBLOCK)
 1498 
 1499     endstdout = endstderr = False
 1500     errlines = []
 1501     while not (endstderr and endstdout):
 1502         lines_before = lines_written
 1503         if not endstdout:
 1504             try:
 1505                 if live:
 1506                     chars = process.stdout.read()
 1507                     if chars == '' and process.poll() is not None:
 1508                         endstdout = True
 1509                     else:
 1510                         fh.write(chars)
 1511                         fh.flush()
 1512                         if '\n' in chars:
 1513                             lines_written += 1
 1514                 else:
 1515                     line = process.stdout.readline()
 1516                     if line == '' and process.poll() is not None:
 1517                         endstdout = True
 1518                     else:
 1519                         fh.write("%s\n" % line.rstrip())
 1520                         fh.flush()
 1521                         lines_written += 1
 1522                         print_line_cr(line, lines_written, stdout=(fh == sys.stdout), tee=tee)
 1523 
 1524             except Exception as ioe:
 1525                 pass
 1526         if not endstderr:
 1527             try:
 1528                 if live:
 1529                     chars = process.stderr.read()
 1530                     if chars == '' and process.poll() is not None:
 1531                         endstderr = True
 1532                     else:
 1533                         fh.write(chars)
 1534                         fh.flush()
 1535                         if '\n' in chars:
 1536                             lines_written += 1
 1537                 else:
 1538                     line = process.stderr.readline()
 1539                     if line == '' and process.poll() is not None:
 1540                         endstderr = True
 1541                     else:
 1542                         errlines.append("%s\n" % line.rstrip())
 1543                         lines_written += 1
 1544                         print_line_cr(line, lines_written, stdout=(fh == sys.stdout), tee=tee)
 1545             except Exception as e:
 1546                 pass
 1547 
 1548         if not lines_written > lines_before:
 1549             # if no output then sleep a bit before checking again
 1550             time.sleep(0.1)
 1551 
 1552     print(" " * 80)
 1553     rc = process.poll()
 1554     if len(errlines) > 0:
 1555         for line in errlines:
 1556             fh.write("%s\n" % line.rstrip())
 1557             fh.flush()
 1558     return rc
 1559 
 1560 
 1561 def is_windows():
 1562     return platform.system().startswith("Win")
 1563 
 1564 
 1565 class Commands(SecretYamlObject):
 1566     yaml_tag = u'!Commands'
 1567     hidden_fields = ['todo_id']
 1568     cmd_continuation_char = "^" if is_windows() else "\\"
 1569     def __init__(self, root_folder, commands_text=None, commands=None, logs_prefix=None, run_text=None, enable_execute=None,
 1570                  confirm_each_command=None, env=None, vars=None, todo_id=None, remove_files=None):
 1571         self.root_folder = root_folder
 1572         self.commands_text = commands_text
 1573         self.vars = vars
 1574         self.env = env
 1575         self.run_text = run_text
 1576         self.remove_files = remove_files
 1577         self.todo_id = todo_id
 1578         self.logs_prefix = logs_prefix
 1579         self.enable_execute = enable_execute
 1580         self.confirm_each_command = confirm_each_command
 1581         self.commands = commands
 1582         for c in self.commands:
 1583             c.todo_id = todo_id
 1584 
 1585     @classmethod
 1586     def from_yaml(cls, loader, node):
 1587         fields = loader.construct_mapping(node, deep = True)
 1588         return Commands(**fields)
 1589 
 1590     def run(self):
 1591         root = self.get_root_folder()
 1592 
 1593         if self.commands_text:
 1594             print(self.get_commands_text())
 1595         if self.env:
 1596             for key in self.env:
 1597                 val = self.jinjaify(self.env[key])
 1598                 os.environ[key] = val
 1599                 if is_windows():
 1600                     print("\n  SET %s=%s" % (key, val))
 1601                 else:
 1602                     print("\n  export %s=%s" % (key, val))
 1603         print(abbreviate_homedir("\n  cd %s" % root))
 1604         commands = ensure_list(self.commands)
 1605         for cmd in commands:
 1606             for line in cmd.display_cmd():
 1607                 print("  %s" % line)
 1608         print()
 1609         if not self.enable_execute is False:
 1610             if self.run_text:
 1611                 print("\n%s\n" % self.get_run_text())
 1612             if len(commands) > 1:
 1613                 if not self.confirm_each_command is False:
 1614                     print("You will get prompted before running each individual command.")
 1615                 else:
 1616                     print(
 1617                         "You will not be prompted for each command but will see the ouput of each. If one command fails the execution will stop.")
 1618             success = True
 1619             if ask_yes_no("Do you want me to run these commands now?"):
 1620                 if self.remove_files:
 1621                     for _f in ensure_list(self.get_remove_files()):
 1622                         f = os.path.join(root, _f)
 1623                         if os.path.exists(f):
 1624                             filefolder = "File" if os.path.isfile(f) else "Folder"
 1625                             if ask_yes_no("%s %s already exists. Shall I remove it now?" % (filefolder, f)) and not dry_run:
 1626                                 if os.path.isdir(f):
 1627                                     shutil.rmtree(f)
 1628                                 else:
 1629                                     os.remove(f)
 1630                 index = 0
 1631                 log_folder = self.logs_prefix if len(commands) > 1 else None
 1632                 for cmd in commands:
 1633                     index += 1
 1634                     if len(commands) > 1:
 1635                         log_prefix = "%02d_" % index
 1636                     else:
 1637                         log_prefix = self.logs_prefix if self.logs_prefix else ''
 1638                     if not log_prefix[-1:] == '_':
 1639                         log_prefix += "_"
 1640                     cwd = root
 1641                     if cmd.cwd:
 1642                         cwd = os.path.join(root, cmd.cwd)
 1643                     folder_prefix = ''
 1644                     if cmd.cwd:
 1645                         folder_prefix = cmd.cwd + "_"
 1646                     if self.confirm_each_command is False or len(commands) == 1 or ask_yes_no("Shall I run '%s' in folder '%s'" % (cmd, cwd)):
 1647                         if self.confirm_each_command is False:
 1648                             print("------------\nRunning '%s' in folder '%s'" % (cmd, cwd))
 1649                         logfilename = cmd.logfile
 1650                         logfile = None
 1651                         cmd_to_run = "%s%s" % ("echo Dry run, command is: " if dry_run else "", cmd.get_cmd())
 1652                         if cmd.redirect:
 1653                             try:
 1654                                 out = run(cmd_to_run, cwd=cwd)
 1655                                 mode = 'a' if cmd.redirect_append is True else 'w'
 1656                                 with open(os.path.join(root, cwd, cmd.get_redirect()), mode) as outfile:
 1657                                     outfile.write(out)
 1658                                     outfile.flush()
 1659                                 print("Wrote %s bytes to redirect file %s" % (len(out), cmd.get_redirect()))
 1660                             except Exception as e:
 1661                                 print("Command %s failed: %s" % (cmd_to_run, e))
 1662                                 success = False
 1663                                 break
 1664                         else:
 1665                             if not cmd.stdout:
 1666                                 if not log_folder:
 1667                                     log_folder = os.path.join(state.get_rc_folder(), "logs")
 1668                                 elif not os.path.isabs(log_folder):
 1669                                     log_folder = os.path.join(state.get_rc_folder(), "logs", log_folder)
 1670                                 if not logfilename:
 1671                                     logfilename = "%s.log" % re.sub(r"\W", "_", cmd.get_cmd())
 1672                                 logfile = os.path.join(log_folder, "%s%s%s" % (log_prefix, folder_prefix, logfilename))
 1673                                 if cmd.tee:
 1674                                     print("Output of command will be printed (logfile=%s)" % logfile)
 1675                                 elif cmd.live:
 1676                                     print("Output will be shown live byte by byte")
 1677                                     logfile = None
 1678                                 else:
 1679                                     print("Wait until command completes... Full log in %s\n" % logfile)
 1680                                 if cmd.comment:
 1681                                     print("# %s\n" % cmd.get_comment())
 1682                             start_time = time.time()
 1683                             returncode = run_with_log_tail(cmd_to_run, cwd, logfile=logfile, tee=cmd.tee, tail_lines=25,
 1684                                                            live=cmd.live, shell=cmd.shell)
 1685                             elapsed = time.time() - start_time
 1686                             if not returncode == 0:
 1687                                 if cmd.should_fail:
 1688                                     print("Command failed, which was expected")
 1689                                     success = True
 1690                                 else:
 1691                                     print("WARN: Command %s returned with error" % cmd.get_cmd())
 1692                                     success = False
 1693                                     break
 1694                             else:
 1695                                 if cmd.should_fail and not dry_run:
 1696                                     print("Expected command to fail, but it succeeded.")
 1697                                     success = False
 1698                                     break
 1699                                 else:
 1700                                     if elapsed > 30:
 1701                                         print("Command completed in %s seconds" % elapsed)
 1702             if not success:
 1703                 print("WARNING: One or more commands failed, you may want to check the logs")
 1704             return success
 1705 
 1706     def get_root_folder(self):
 1707         return self.jinjaify(self.root_folder)
 1708 
 1709     def get_commands_text(self):
 1710         return self.jinjaify(self.commands_text)
 1711 
 1712     def get_run_text(self):
 1713         return self.jinjaify(self.run_text)
 1714 
 1715     def get_remove_files(self):
 1716         return self.jinjaify(self.remove_files)
 1717 
 1718     def get_vars(self):
 1719         myvars = {}
 1720         if self.vars:
 1721             for k in self.vars:
 1722                 val = self.vars[k]
 1723                 if callable(val):
 1724                     myvars[k] = expand_jinja(val(), vars=myvars)
 1725                 else:
 1726                     myvars[k] = expand_jinja(val, vars=myvars)
 1727         return myvars
 1728 
 1729     def jinjaify(self, data, join=False):
 1730         if not data:
 1731             return None
 1732         v = self.get_vars()
 1733         if self.todo_id:
 1734             v.update(state.get_todo_by_id(self.todo_id).get_vars())
 1735         if isinstance(data, list):
 1736             if join:
 1737                 return expand_jinja(" ".join(data), v)
 1738             else:
 1739                 res = []
 1740                 for rf in data:
 1741                     res.append(expand_jinja(rf, v))
 1742                 return res
 1743         else:
 1744             return expand_jinja(data, v)
 1745 
 1746 
 1747 def abbreviate_homedir(line):
 1748     if is_windows():
 1749         if 'HOME' in os.environ:
 1750             return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%HOME%", line)
 1751         elif 'USERPROFILE' in os.environ:
 1752             return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%USERPROFILE%", line)
 1753     else:
 1754         return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1~", line)
 1755 
 1756 
 1757 class Command(SecretYamlObject):
 1758     yaml_tag = u'!Command'
 1759     hidden_fields = ['todo_id']
 1760     def __init__(self, cmd, cwd=None, stdout=None, logfile=None, tee=None, live=None, comment=None, vars=None,
 1761                  todo_id=None, should_fail=None, redirect=None, redirect_append=None, shell=None):
 1762         self.cmd = cmd
 1763         self.cwd = cwd
 1764         self.comment = comment
 1765         self.logfile = logfile
 1766         self.vars = vars
 1767         self.tee = tee
 1768         self.live = live
 1769         self.stdout = stdout
 1770         self.should_fail = should_fail
 1771         self.shell = shell
 1772         self.todo_id = todo_id
 1773         self.redirect_append = redirect_append
 1774         self.redirect = redirect
 1775         if tee and stdout:
 1776             self.stdout = None
 1777             print("Command %s specifies 'tee' and 'stdout', using only 'tee'" % self.cmd)
 1778         if live and stdout:
 1779             self.stdout = None
 1780             print("Command %s specifies 'live' and 'stdout', using only 'live'" % self.cmd)
 1781         if live and tee:
 1782             self.tee = None
 1783             print("Command %s specifies 'tee' and 'live', using only 'live'" % self.cmd)
 1784         if redirect and (tee or stdout or live):
 1785             self.tee = self.stdout = self.live = None
 1786             print("Command %s specifies 'redirect' and other out options at the same time. Using redirect only" % self.cmd)
 1787 
 1788     @classmethod
 1789     def from_yaml(cls, loader, node):
 1790         fields = loader.construct_mapping(node, deep = True)
 1791         return Command(**fields)
 1792 
 1793     def get_comment(self):
 1794         return self.jinjaify(self.comment)
 1795 
 1796     def get_redirect(self):
 1797         return self.jinjaify(self.redirect)
 1798 
 1799     def get_cmd(self):
 1800         return self.jinjaify(self.cmd, join=True)
 1801 
 1802     def get_vars(self):
 1803         myvars = {}
 1804         if self.vars:
 1805             for k in self.vars:
 1806                 val = self.vars[k]
 1807                 if callable(val):
 1808                     myvars[k] = expand_jinja(val(), vars=myvars)
 1809                 else:
 1810                     myvars[k] = expand_jinja(val, vars=myvars)
 1811         return myvars
 1812 
 1813     def __str__(self):
 1814         return self.get_cmd()
 1815 
 1816     def jinjaify(self, data, join=False):
 1817         v = self.get_vars()
 1818         if self.todo_id:
 1819             v.update(state.get_todo_by_id(self.todo_id).get_vars())
 1820         if isinstance(data, list):
 1821             if join:
 1822                 return expand_jinja(" ".join(data), v)
 1823             else:
 1824                 res = []
 1825                 for rf in data:
 1826                     res.append(expand_jinja(rf, v))
 1827                 return res
 1828         else:
 1829             return expand_jinja(data, v)
 1830 
 1831     def display_cmd(self):
 1832         lines = []
 1833         pre = post = ''
 1834         if self.comment:
 1835             if is_windows():
 1836                 lines.append("REM %s" % self.get_comment())
 1837             else:
 1838                 lines.append("# %s" % self.get_comment())
 1839         if self.cwd:
 1840             lines.append("pushd %s" % self.cwd)
 1841         redir = "" if self.redirect is None else " %s %s" % (">" if self.redirect_append is None else ">>" , self.get_redirect())
 1842         line = "%s%s" % (expand_multiline(self.get_cmd(), indent=2), redir)
 1843         # Print ~ or %HOME% rather than the full expanded homedir path
 1844         line = abbreviate_homedir(line)
 1845         lines.append(line)
 1846         if self.cwd:
 1847             lines.append("popd")
 1848         return lines
 1849 
 1850 class UserInput(SecretYamlObject):
 1851     yaml_tag = u'!UserInput'
 1852 
 1853     def __init__(self, name, prompt, type=None):
 1854         self.type = type
 1855         self.prompt = prompt
 1856         self.name = name
 1857 
 1858     @classmethod
 1859     def from_yaml(cls, loader, node):
 1860         fields = loader.construct_mapping(node, deep = True)
 1861         return UserInput(**fields)
 1862 
 1863     def run(self, dict=None):
 1864         correct = False
 1865         while not correct:
 1866             try:
 1867                 result = str(input("%s : " % self.prompt))
 1868                 if self.type and self.type == 'int':
 1869                     result = int(result)
 1870                 correct = True
 1871             except Exception as e:
 1872                 print("Incorrect input: %s, try again" % e)
 1873                 continue
 1874             if dict:
 1875                 dict[self.name] = result
 1876             return result
 1877 
 1878 
 1879 def create_ical(todo):
 1880     if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
 1881         c = Calendar()
 1882         e = Event()
 1883         e.name = "Lucene/Solr %s vote ends" % state.release_version
 1884         e.begin = vote_close_72h_date()
 1885         e.description = "Remember to sum up votes and continue release :)"
 1886         c.events.add(e)
 1887         ics_file = os.path.join(state.get_rc_folder(), 'vote_end.ics')
 1888         with open(ics_file, 'w') as my_file:
 1889             my_file.writelines(c)
 1890         open_file(ics_file)
 1891     return True
 1892 
 1893 
 1894 today = datetime.utcnow().date()
 1895 sundays = {(today + timedelta(days=x)): 'Sunday' for x in range(10) if (today + timedelta(days=x)).weekday() == 6}
 1896 y = datetime.utcnow().year
 1897 years = [y, y+1]
 1898 non_working = holidays.CA(years=years) + holidays.US(years=years) + holidays.England(years=years) \
 1899               + holidays.DE(years=years) + holidays.NO(years=years) + holidays.IND(years=years) + holidays.RU(years=years)
 1900 
 1901 
 1902 def vote_close_72h_date():
 1903     # Voting open at least 72 hours according to ASF policy
 1904     return datetime.utcnow() + timedelta(hours=73)
 1905 
 1906 
 1907 def vote_close_72h_holidays():
 1908     days = 0
 1909     day_offset = -1
 1910     holidays = []
 1911     # Warn RM about major holidays coming up that should perhaps extend the voting deadline
 1912     # Warning will be given for Sunday or a public holiday observed by 3 or more [CA, US, EN, DE, NO, IND, RU]
 1913     while days < 3:
 1914         day_offset += 1
 1915         d = today + timedelta(days=day_offset)
 1916         if not (d in sundays or (d in non_working and len(non_working[d]) >= 2)):
 1917             days += 1
 1918         else:
 1919             if d in sundays:
 1920                 holidays.append("%s (Sunday)" % d)
 1921             else:
 1922                 holidays.append("%s (%s)" % (d, non_working[d]))
 1923     return holidays if len(holidays) > 0 else None
 1924 
 1925 
 1926 def website_javadoc_redirect(todo):
 1927     htfile = os.path.join(website_folder, 'content', '.htaccess')
 1928     latest = state.get_latest_version()
 1929     if Version.parse(state.release_version).gt(Version.parse(latest)):
 1930         print("We are releasing the latest version ")
 1931         htaccess = file_to_string(htfile)
 1932         print("NOT YET IMPLEMENTED")
 1933         return False
 1934     else:
 1935         print("Task not necessary since %s is not the latest release version" % state.release_version)
 1936         return True
 1937 
 1938 
 1939 def prepare_highlights(todo):
 1940     if not os.path.exists(lucene_highlights_file):
 1941         with open(lucene_highlights_file, 'w') as fp:
 1942             fp.write("* New cool Lucene feature\n* Important bugfix")
 1943     if not os.path.exists(solr_highlights_file):
 1944         with open(solr_highlights_file, 'w') as fp:
 1945             fp.write("* New cool Solr feature\n* Important bugfix")
 1946     return True
 1947 
 1948 
 1949 def prepare_announce(todo):
 1950     if not os.path.exists(tlp_news_draft):
 1951         tlp_text = expand_jinja("(( template=announce_tlp ))")
 1952         with open(tlp_news_draft, 'w') as fp:
 1953             fp.write(tlp_text)
 1954         # print("Wrote TLP announce draft to %s" % tlp_news_file)
 1955 
 1956         lucene_text = expand_jinja("(( template=announce_lucene ))")
 1957         with open(lucene_news_draft, 'w') as fp:
 1958             fp.write(lucene_text)
 1959         # print("Wrote Lucene announce draft to %s" % lucene_news_file)
 1960 
 1961         solr_text = expand_jinja("(( template=announce_solr ))")
 1962         with open(solr_news_draft, 'w') as fp:
 1963             fp.write(solr_text)
 1964         # print("Wrote Solr announce draft to %s" % solr_news_file)
 1965     else:
 1966         print("Drafts already exist, not re-generating")
 1967     return True
 1968 
 1969 
 1970 def patch_news_file(orig, draft, sticky_lines):
 1971     orig_lines = open(orig).readlines()
 1972     draft_lines = open(draft).readlines()
 1973     lines = orig_lines[0:sticky_lines]
 1974     lines.extend(draft_lines)
 1975     lines.append('\n')
 1976     lines.append('\n')
 1977     lines.extend(orig_lines[sticky_lines:])
 1978     with open(orig, 'w') as fp:
 1979         fp.writelines(lines)
 1980     print("Added news to %s" % orig)
 1981 
 1982 
 1983 def update_news(todo):
 1984     touch_file = os.path.join(state.get_release_folder(), 'news_updated')
 1985     if not os.path.exists(touch_file):
 1986         patch_news_file(tlp_news_file, tlp_news_draft, 2)
 1987         patch_news_file(lucene_news_file, lucene_news_draft, 2)
 1988         patch_news_file(solr_news_file, solr_news_draft, 4)
 1989 
 1990         latest = state.get_latest_version()
 1991         if Version.parse(state.release_version).gt(Version.parse(latest)):
 1992             print("We are releasing the latest version, updating latestversion.mdtext")
 1993             with open(os.path.join(website_folder, 'content', 'latestversion.mdtext'), 'w') as fp:
 1994                 fp.write(state.release_version)
 1995 
 1996         with open(touch_file, 'w') as fp:
 1997             fp.write("true")
 1998         print("News files in website folder updated with draft announcements")
 1999     else:
 2000         print("News files not changed, already patched earlier. Please edit by hand")
 2001 
 2002     return True
 2003 
 2004 
 2005 def set_java_home(version):
 2006     os.environ['JAVA_HOME'] = state.get_java_home_for_version(version)
 2007     os.environ['JAVACMD'] = state.get_java_cmd_for_version(version)
 2008 
 2009 
 2010 def load_lines(file):
 2011     if os.path.exists(file):
 2012         with open(file, 'r') as fp:
 2013             return fp.readlines()
 2014     else:
 2015         return ['* foo', '* bar']
 2016 
 2017 
 2018 if __name__ == '__main__':
 2019     try:
 2020         main()
 2021     except KeyboardInterrupt:
 2022         print('Keyboard interrupt...exiting')