Packaging python scripts

dev/before_install

@@ -2,15 +2,15 @@
 
 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
 	sudo apt-get update -qq
-	sudo apt-get install -qq cvs git mercurial cssc bzr subversion monotone rcs rcs-blame python3 python3-pip pep8 nodejs;
+	sudo apt-get install -qq cvs git mercurial cssc bzr subversion monotone rcs rcs-blame python3 python3-venv python3-pip pep8 nodejs;
 	sudo ./dev/install-bitkeeper.sh
 	sudo pip3 install --upgrade pip
 	sudo pip3 install flake8
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
 	brew update
 	brew install ctags cvs node
 	brew upgrade python
-	pip3 install pep8 flake8
+	pip3 install pep8 flake8 virtualenv
 fi
 
 sudo ./dev/install-universal_ctags.sh

distribution/assembly.xml

@@ -19,6 +19,11 @@
         <outputDirectory>lib</outputDirectory>
         <destName>source.war</destName>
       </file>
+      <file>
+        <source>${project.basedir}/../opengrok-tools/dist/opengrok-tools-${parsedVersion.majorVersion}.${parsedVersion.minorVersion}${parsedVersion.qualifier}.tar.gz</source>
+        <outputDirectory>tools</outputDirectory>
+        <destName>opengrok-tools.tar.gz</destName>
+      </file>
     </files>
     <fileSets>
         <fileSet>
@@ -39,13 +44,6 @@
             <include>opengrok.1</include>
           </includes>
         </fileSet>
-        <fileSet>
-          <directory>${project.basedir}/../tools/src/main/python</directory>
-          <outputDirectory>bin</outputDirectory>
-          <includes>
-            <include>*.py</include>
-          </includes>
-        </fileSet>
         <fileSet>
           <directory>${project.build.directory}/../lib</directory>
           <outputDirectory>lib/lib</outputDirectory>

distribution/pom.xml

@@ -36,7 +36,7 @@
 
     <dependency>
       <groupId>org.opengrok</groupId>
-      <artifactId>tools</artifactId>
+      <artifactId>opengrok-tools</artifactId>
       <version>1.1-rc58</version>
       <type>pom</type>
     </dependency>
@@ -64,6 +64,19 @@
               </execution>
             </executions>
           </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <version>3.0.0</version>
+            <executions>
+              <execution>
+                <id>parse-version</id>
+                <goals>
+                  <goal>parse-version</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>

opengrok-tools/.gitignore

@@ -0,0 +1,4 @@
+env
+build
+dist
+opengrok_tools.egg-info

opengrok-tools/MANIFEST.in

@@ -0,0 +1 @@
+include src/main/python/README.md

opengrok-tools/pom.xml

@@ -0,0 +1,139 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+CDDL HEADER START
+
+The contents of this file are subject to the terms of the
+Common Development and Distribution License (the "License").
+You may not use this file except in compliance with the License.
+
+See LICENSE.txt included in this distribution for the specific
+language governing permissions and limitations under the License.
+
+When distributing Covered Code, include this CDDL HEADER in each
+file and include the License file at LICENSE.txt.
+If applicable, add the following below this CDDL HEADER, with the
+fields enclosed by brackets "[]" replaced with your own identifying
+information: Portions Copyright [yyyy] [name of copyright owner]
+
+CDDL HEADER END
+
+Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved.
+Portions Copyright (c) 2017-2018, Chris Fraire <[email protected]>.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.opengrok</groupId>
+        <artifactId>opengrok-top</artifactId>
+        <version>1.1-rc58</version>
+    </parent>
+
+    <artifactId>opengrok-tools</artifactId>
+    <version>1.1-rc58</version>
+    <packaging>pom</packaging>
+
+    <name>OpenGrok tools</name>
+
+    <build>
+        <sourceDirectory>src/main/python</sourceDirectory>
+        <testSourceDirectory>src/test/python</testSourceDirectory>
+
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>1.6.0</version>
+                <executions>
+                    <execution>
+                        <configuration>
+                            <executable>flake8</executable>
+                            <arguments>
+                                <argument>-v</argument>
+                                <argument>--exclude=filelock.py,test_command.py,test_commands.py</argument>
+                                <argument>${project.build.sourceDirectory}</argument>
+                                <argument>setup.py</argument>
+                            </arguments>
+                        </configuration>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>generate python env</id>
+                        <configuration>
+                            <executable>python3</executable>
+                            <workingDirectory>${project.build.directory}/../</workingDirectory>
+                            <arguments>
+                                <argument>-m</argument>
+                                <argument>venv</argument>
+                                <argument>env</argument>
+                            </arguments>
+                        </configuration>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>generate-package</id>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>env/bin/python</executable>
+                            <workingDirectory>${project.build.directory}/../</workingDirectory>
+                            <arguments>
+                                <argument>setup.py</argument>
+                                <argument>sdist</argument>
+                            </arguments>
+                        </configuration>
+                        <phase>package</phase>
+                    </execution>
+                    <execution>
+                        <id>pre-python-test</id>
+                        <configuration>
+                            <executable>env/bin/python</executable>
+                            <workingDirectory>${project.build.directory}/../</workingDirectory>
+                            <arguments>
+                                <argument>setup.py</argument>
+                                <argument>install</argument>
+                            </arguments>
+                        </configuration>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>python-test</id>
+                        <configuration>
+                            <executable>env/bin/python</executable>
+                            <workingDirectory>${project.build.directory}/../</workingDirectory>
+                            <arguments>
+                                <argument>setup.py</argument>
+                                <argument>test</argument>
+                            </arguments>
+                        </configuration>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                    </execution>
+                </executions>
+
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+            </plugin>
+
+        </plugins>
+    </build>
+
+</project>

opengrok-tools/setup.py

@@ -0,0 +1,63 @@
+import os
+import unittest
+from setuptools import setup
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def readme():
+    with open(os.path.join(SCRIPT_DIR, 'src', 'main', 'python', 'README.md'),
+              'r') as readme:
+        return readme.read()
+
+
+def my_test_suite():
+    test_loader = unittest.TestLoader()
+    test_suite = test_loader.discover(
+        os.path.join(SCRIPT_DIR, 'src', 'test', 'python'), pattern='test_*.py')
+    return test_suite
+
+
+setup(
+    name='opengrok-tools',
+    version='1.1rc58',
+    packages=[
+        'opengrok_tools',
+        'all',
+        'all.utils',
+        'all.scm',
+    ],
+    package_dir={
+        'opengrok_tools': 'src/main/python/opengrok_tools',
+        'all': 'src/main/python/opengrok_tools/all',
+        'all.scm': 'src/main/python/opengrok_tools/all/scm',
+        'all.utils': 'src/main/python/opengrok_tools/all/utils',
+    },
+    url='https://github.com/OpenGrok/OpenGrok',
+    license='CDDL',
+    author='Oracle',
+    author_email='[email protected]',
+    description='Tools for managing OpenGrok instance',
+    long_description=readme(),
+    test_suite='setup.my_test_suite',
+    install_requires=[
+        'jsonschema==2.6.0',
+        'pyyaml',
+        'requests',
+        'resource'
+    ],
+    entry_points={
+        'console_scripts': [
+            'opengrok-config-merge=opengrok_tools.config_merge:main',
+            'opengrok-deploy=opengrok_tools.deploy:main',
+            'opengrok-groups=opengrok_tools.groups:main',
+            'opengrok=opengrok_tools.indexer:main',
+            'opengrok-indexer=opengrok_tools.indexer:main',
+            'opengrok-java=opengrok_tools.java:main',
+            'opengrok-mirror=opengrok_tools.mirror:main',
+            'opengrok-projadm=opengrok_tools.projadm:main',
+            'opengrok-reindex-project=opengrok_tools.reindex_project:main',
+            'opengrok-sync=opengrok_tools.sync:main',
+        ]
+    },
+)

opengrok-tools/src/main/python/README.md

@@ -0,0 +1,105 @@
+
+# OpenGrok tools
+
+Set of scripts to facilitate project synchronization and mirroring
+
+The scripts require Python 3 and they rely on a binary/symlink `python3` to be
+present that points to the latest Python 3.x version present on the system.
+
+See https://github.com/oracle/opengrok/wiki/Repository-synchronization
+for more details.
+
+# Content
+
+This is a list of the binaries in this package.
+
+```text
+opengrok-config-merge
+opengrok-deploy
+opengrok-groups
+opengrok
+opengrok-indexer
+opengrok-java
+opengrok-mirror
+opengrok-projadm
+opengrok-reindex-project
+opengrok-sync
+```
+
+# Development
+
+## Environment
+
+Prepare a virtual environment
+
+```bash
+python3 -m venv env
+. env/bin/activate
+```
+
+Install development dependencies
+
+```bash
+python -m pip install -r requirements.txt
+```
+
+## Developing
+
+Start developing, making changes in files. Test your changes with usual python commands.
+
+```bash
+python src/main/python/opengrok_tools/groups.py
+python src/main/python/opengrok_tools/sync.py
+```
+
+## Installation
+
+Test installing your package into the local environment
+
+```bash
+python setup.py install
+# now you can try console scripts
+opengrok-groups
+opengrok-sync
+```
+
+or make a distribution tarball.
+
+```bash
+python setup.py sdist
+ls -l dist/
+```
+
+### Installation on the target system
+
+Use the distribution tarball and run `pip`.
+
+```bash
+python3 -m pip install opengrok_tools-{version}.tar.gz
+```
+
+This will download all dependencies and install the package to your local python3 modules.
+You can use console scripts to run the package binaries.
+
+#### Uninstalling
+
+```bash
+python3 -m pip uninstall opengrok_tools
+```
+
+## Testing
+
+```bash
+python setup.py install test
+mvn test
+```
+
+## Cleanup
+
+Deactivate the virtual environment
+```bash
+deactivate
+# optionally
+# rm -r env
+```
+