Add a "single" source/header file generation and build mode (#2790)

Introduce a new way of building: before any file is compiled
all source files are combined into a single C file and all
header files into a single H file (per subdir).

This new approach makes it possible to quickly integrate JerryScript
into other projects:

```
$ gcc -o demo demo.c jerryscript.c jerryscript-port-default.c -lm
```

To use the source generator run:
```
$ cmake -Bbuild_dir -H. -DENABLE_ALL_IN_ONE_SOURCE=ON
$ make -C build_dir generate-single-source
```

This will create the following files in the `build_dir`:
* jerryscript.c
* jerryscript.h
* jerryscript-config.h
* jerryscript-port-default.c
* jerryscript-port-default.h

JerryScript-DCO-1.0-Signed-off-by: Peter Gal pgal.u-szeged@partner.samsung.com

jerry-core/CMakeLists.txt

@@ -150,6 +150,56 @@ if(ENABLE_ALL_IN_ONE)
   set(SOURCE_CORE_FILES ${ALL_IN_FILE})
 endif()
 
+# "Single" JerryScript source/header build.
+#  The process will create the following files:
+#   * jerryscript.c
+#   * jerryscript.h
+#   * jerryscript-config.h
+if(ENABLE_ALL_IN_ONE_SOURCE)
+  # Create a default configuration
+  set(JERRYSCRIPT_CONFIG_H "${CMAKE_BINARY_DIR}/jerryscript-config.h")
+  set(JERRYSCRIPT_SOURCE_CONFIG_H "${CMAKE_CURRENT_SOURCE_DIR}/config.h")
+  add_custom_command(OUTPUT ${JERRYSCRIPT_CONFIG_H}
+                     COMMAND ${CMAKE_COMMAND} -E copy ${JERRYSCRIPT_SOURCE_CONFIG_H} ${JERRYSCRIPT_CONFIG_H}
+                     DEPENDS ${JERRYSCRIPT_SOURCE_CONFIG_H})
+
+  # Create single C file
+  file(GLOB HEADER_CORE_FILES *.h)
+  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/jerryscript.c")
+  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/jerryscript.h")
+  add_custom_command(OUTPUT ${ALL_IN_FILE}
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+                             --base-dir ${CMAKE_CURRENT_SOURCE_DIR}
+                             --input ${CMAKE_CURRENT_SOURCE_DIR}/api/jerry.c
+                             --output ${ALL_IN_FILE}
+                             --append-c-files
+                             --remove-include jerryscript.h
+                             --remove-include jerryscript-port.h
+                             --remove-include jerryscript-compiler.h
+                             --remove-include jerryscript-core.h
+                             --remove-include jerryscript-debugger.h
+                             --remove-include jerryscript-debugger-transport.h
+                             --remove-include jerryscript-port.h
+                             --remove-include jerryscript-snapshot.h
+                             --remove-include config.h
+                             --push-include jerryscript.h
+                    DEPENDS ${SOURCE_CORE_FILES} ${ALL_IN_FILE_H} ${JERRYSCRIPT_CONFIG_H}
+  )
+  add_custom_command(OUTPUT ${ALL_IN_FILE_H}
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+                             --base-dir ${CMAKE_CURRENT_SOURCE_DIR}
+                             --input ${CMAKE_CURRENT_SOURCE_DIR}/include/jerryscript.h
+                             --output ${ALL_IN_FILE_H}
+                             --remove-include config.h
+                             --push-include jerryscript-config.h
+                     DEPENDS ${HEADER_CORE_FILES} ${JERRYSCRIPT_CONFIG_H}
+  )
+  add_custom_target(generate-single-source-jerry DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+  add_custom_target(generate-single-source DEPENDS generate-single-source-jerry)
+
+  set(SOURCE_CORE_FILES ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+endif()
+
 # Third-party
 # Valgrind
 set(INCLUDE_THIRD_PARTY_VALGRIND "${CMAKE_SOURCE_DIR}/third-party/valgrind")

jerry-port/default/CMakeLists.txt

@@ -22,6 +22,43 @@ set(INCLUDE_PORT_DEFAULT "${CMAKE_CURRENT_SOURCE_DIR}/include")
 # Source directories
 file(GLOB SOURCE_PORT_DEFAULT *.c)
 
+# "Single" JerryScript source/header build.
+#  The process will create the following files:
+#   * jerryscript-port-default.c
+#   * jerryscript-port-default.h
+if(ENABLE_ALL_IN_ONE_SOURCE)
+  file(GLOB HEADER_PORT_DEFAULT *.h)
+  set(ALL_IN_FILE "${CMAKE_BINARY_DIR}/jerryscript-port-default.c")
+  set(ALL_IN_FILE_H "${CMAKE_BINARY_DIR}/jerryscript-port-default.h")
+  add_custom_command(OUTPUT ${ALL_IN_FILE}
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+                             --base-dir ${CMAKE_CURRENT_SOURCE_DIR}
+                             --output ${ALL_IN_FILE}
+                             --append-c-files
+                             --remove-include jerryscript-port.h
+                             --remove-include jerryscript-port-default.h
+                             --remove-include jerryscript-debugger.h
+                             --push-include jerryscript.h
+                             --push-include jerryscript-port-default.h
+                     DEPENDS ${SOURCE_PORT_DEFAULT} ${ALL_IN_FILE_H}
+  )
+
+  add_custom_command(OUTPUT ${ALL_IN_FILE_H}
+                     COMMAND python ${CMAKE_SOURCE_DIR}/tools/srcmerger.py
+                               --base-dir ${CMAKE_CURRENT_SOURCE_DIR}/
+                               --input ${CMAKE_CURRENT_SOURCE_DIR}/include/jerryscript-port-default.h
+                               --output ${ALL_IN_FILE_H}
+                               --remove-include jerryscript-port.h
+                               --remove-include jerryscript.h
+                               --push-include jerryscript.h
+                    DEPENDS ${HEADER_PORT_DEFAULT}
+  )
+  add_custom_target(generate-single-source-port DEPENDS ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+  add_dependencies(generate-single-source generate-single-source-port)
+
+  set(SOURCE_PORT_DEFAULT ${ALL_IN_FILE} ${ALL_IN_FILE_H})
+endif()
+
 # Define _BSD_SOURCE and _DEFAULT_SOURCE
 # (should only be necessary if we used compiler default libc but not checking that)
 set(DEFINES_PORT_DEFAULT _BSD_SOURCE _DEFAULT_SOURCE)

tools/run-tests.py

@@ -160,6 +160,8 @@ JERRY_BUILDOPTIONS = [
             ['--regexp-recursion-limit=1000']),
     Options('buildoption_test-vm_recursion_limit',
             OPTIONS_VM_RECURSION_LIMIT),
+    Options('buildoption_test-single-source',
+            ['--cmake-param=-DENABLE_ALL_IN_ONE_SOURCE=ON']),
 ]
 
 def get_arguments():

tools/srcmerger.py

+#!/usr/bin/env python
+
+# Copyright JS Foundation and other contributors, http://js.foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import argparse
+import fnmatch
+import logging
+import os
+import re
+import sys
+
+
+class SourceMerger(object):
+    # pylint: disable=too-many-instance-attributes
+
+    _RE_INCLUDE = re.compile(r'\s*#include ("|<)(.*?)("|>)\n$')
+
+    def __init__(self, h_files, extra_includes=None, remove_includes=None):
+        self._log = logging.getLogger('sourcemerger')
+        self._last_builtin = None
+        self._processed = []
+        self._output = []
+        self._h_files = h_files
+        self._extra_includes = extra_includes or []
+        self._remove_includes = remove_includes
+        # The copyright will be loaded from the first input file
+        self._copyright = {'lines': [], 'loaded': False}
+
+    def _process_non_include(self, line, file_level):
+        # Special case #2: Builtin include header name usage
+        if line.strip() == "#include BUILTIN_INC_HEADER_NAME":
+            assert self._last_builtin is not None, 'No previous BUILTIN_INC_HEADER_NAME definition'
+            self._log.debug('[%d] Detected usage of BUILTIN_INC_HEADER_NAME, including: %s',
+                            file_level, self._last_builtin)
+            self.add_file(self._h_files[self._last_builtin])
+            # return from the function as we have processed the included file
+            return
+
+        # Special case #1: Builtin include header name definition
+        if line.startswith('#define BUILTIN_INC_HEADER_NAME '):
+            # the line is in this format: #define BUILTIN_INC_HEADER_NAME "<filename>"
+            self._last_builtin = line.split('"', 2)[1]
+            self._log.debug('[%d] Detected definition of BUILTIN_INC_HEADER_NAME: %s',
+                            file_level, self._last_builtin)
+
+        # the line is not anything special, just push it into the output
+        self._output.append(line)
+
+    def add_file(self, filename, file_level=0):
+        if os.path.basename(filename) in self._processed:
+            self._log.warning('Tried to to process an already processed file: "%s"', filename)
+            return
+
+        file_level += 1
+
+        # mark the start of the new file in the output
+        self._output.append('#line 1 "%s"\n' % (filename))
+
+        line_idx = 0
+        with open(filename, 'r') as input_file:
+            in_copyright = False
+            for line in input_file:
+                line_idx += 1
+
+                if not in_copyright and line.startswith('/* Copyright '):
+                    in_copyright = True
+                    if not self._copyright['loaded']:
+                        self._copyright['lines'].append(line)
+                    continue
+
+                if in_copyright:
+                    if not self._copyright['loaded']:
+                        self._copyright['lines'].append(line)
+
+                    if line.strip().endswith('*/'):
+                        in_copyright = False
+                        self._copyright['loaded'] = True
+
+                    continue
+
+                # check if the line is an '#include' line
+                match = SourceMerger._RE_INCLUDE.match(line)
+                if not match:
+                    # the line is not a header
+                    self._process_non_include(line, file_level)
+                    continue
+
+                if match.group(1) == '<':
+                    # found a "global" include
+                    self._output.append(line)
+                    continue
+
+                name = match.group(2)
+
+                if name in self._remove_includes:
+                    self._log.debug('[%d] Removing include line (%s:%d): %s',
+                                    file_level, filename, line_idx, line.strip())
+                    continue
+
+                if name not in self._h_files:
+                    self._log.warning('[%d] Include not found: "%s" in "%s:%d"',
+                                      file_level, name, filename, line_idx)
+                    self._output.append(line)
+                    continue
+
+                if name in self._processed:
+                    self._log.debug('[%d] Already included: "%s"',
+                                    file_level, name)
+                    continue
+
+                self._log.debug('[%d] Including: "%s"',
+                                file_level, self._h_files[name])
+                self.add_file(self._h_files[name])
+
+                # mark the continuation of the current file in the output
+                self._output.append('#line %d "%s"\n' % (line_idx + 1, filename))
+
+                if not name.endswith('.inc.h'):
+                    # if the included file is not a "*.inc.h" file mark it as processed
+                    self._processed.append(name)
+
+        file_level -= 1
+        if not filename.endswith('.inc.h'):
+            self._processed.append(os.path.basename(filename))
+
+    def write_output(self, out_fp):
+        for line in self._copyright['lines']:
+            out_fp.write(line)
+
+        for include in self._extra_includes:
+            out_fp.write('#include "%s"\n' % include)
+
+        for line in self._output:
+            out_fp.write(line)
+
+
+def match_files(base_dir, pattern):
+    """
+    Return the files matching the given pattern.
+
+    :param base_dir: directory to search in
+    :param pattern: file pattern to use
+    :returns generator: the generator which iterates the matching file names
+    """
+    for path, _, files in os.walk(base_dir):
+        for name in files:
+            if fnmatch.fnmatch(name, pattern):
+                yield os.path.join(path, name)
+
+
+def collect_files(base_dir, pattern):
+    """
+    Collect files in the provided base directory given a file pattern.
+    Will collect all files in the base dir recursively.
+
+    :param base_dir: directory to search in
+    :param pattern: file pattern to use
+    :returns dictionary: a dictionary file base name -> file path mapping
+    """
+    name_mapping = {}
+    for fname in match_files(base_dir, pattern):
+        name = os.path.basename(fname)
+
+        if name in name_mapping:
+            print('Duplicate name detected: "%s" and "%s"' % (name, name_mapping[name]))
+            continue
+
+        name_mapping[name] = fname
+
+    return name_mapping
+
+
+def run_merger(args):
+    h_files = collect_files(args.base_dir, '*.h')
+    c_files = collect_files(args.base_dir, '*.c')
+
+    for name in args.remove_include:
+        c_files.pop(name, '')
+        h_files.pop(name, '')
+
+    merger = SourceMerger(h_files, args.push_include, args.remove_include)
+    if args.input_file:
+        merger.add_file(args.input_file)
+
+    if args.append_c_files:
+        # if the input file is in the C files list it should be removed to avoid
+        # double inclusion of the file
+        if args.input_file:
+            input_name = os.path.basename(args.input_file)
+            c_files.pop(input_name, '')
+
+        # Add the C files in reverse the order to make sure that builtins are
+        # not at the beginning.
+        for fname in sorted(c_files.values(), reverse=True):
+            merger.add_file(fname)
+
+    with open(args.output_file, 'w') as output:
+        merger.write_output(output)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Merge source/header files.')
+    parser.add_argument('--base-dir', metavar='DIR', type=str, dest='base_dir',
+                        help='', default=os.path.curdir)
+    parser.add_argument('--input', metavar='FILE', type=str, dest='input_file',
+                        help='Main input source/header file')
+    parser.add_argument('--output', metavar='FILE', type=str, dest='output_file',
+                        help='Output source/header file')
+    parser.add_argument('--append-c-files', dest='append_c_files', default=False,
+                        action='store_true', help='das')
+    parser.add_argument('--remove-include', action='append', default=[])
+    parser.add_argument('--push-include', action='append', default=[])
+    parser.add_argument('--verbose', '-v', action='store_true', default=False)
+
+    args = parser.parse_args()
+
+    log_level = logging.WARNING
+    if args.verbose:
+        log_level = logging.DEBUG
+
+    logging.basicConfig(level=log_level)
+    logging.debug('Starting merge with args: %s', str(sys.argv))
+
+    run_merger(args)
+
+
+if __name__ == "__main__":
+    main()