ENH: make it easier to switch between the various make dependencies programs.

- However, the new ragel-based parser is much faster
  than the others, and does not cause 'too many open files' error
  that the flex-based parser does (issue #784).

  The timings (using src/sampling as being somewhat representative)

    $ wclean; wmakeLnInclude -u .; time wmake -s dep

        3.4s  wmkdepend (ragel) [now default]
        5.7s  wmkdep (flex)
        6.1s  cpp -M

- The makeDepend script is for testing purposes only, but could used as
  a hook for other dependency generation systems (eg, ninja).
  It simply wraps 'cpp -M' in a form that is calling compatible with
  wmkdepend.

BUG: wmkdepend parser was missing optional leading space on #include match

STYLE: use -G2 (goto-based) option for wmkdepend state machine

- the machine is compact with few states and lends itself to this

wmake/rules/General/transform

@@ -1,5 +1,11 @@
 #----------------------------*- makefile-gmake -*------------------------------
 
+# The dependency generation program
+
+WMKDEP  := $(WMAKE_BIN)/wmkdepend    # 1. Ragel-based
+# WMKDEP  := $(WMAKE_BIN)/wmkdep       # 2. Flex-based (slower, ulimit problem)
+# WMKDEP  := $(WM_SCRIPTS)/makeDepend  # 3. cpp -M (for testing only)
+
 WMKDEP_FLAGS := -eWM_PROJECT_DIR -eWM_THIRD_PARTY_DIR
 
 ifneq ("$(WM_QUIET)","")
@@ -31,7 +37,6 @@ $(OBJECTS_DIR)/%.dep : %
 	$(call QUIET_MESSAGE,dep,$(<F))
 	$(call VERBOSE_MESSAGE,Making dependency list for source file,$(<F))
 	@$(WM_SCRIPTS)/makeTargetDir $@
-	@$(WMAKE_BIN)/wmkdepend $(WMKDEP_FLAGS) -o$@ -I$(*D) $(LIB_HEADER_DIRS) $<
+	@$(WMKDEP) $(WMKDEP_FLAGS) -o$@ -I$(*D) $(LIB_HEADER_DIRS) $<
-#flex	@$(WMAKE_BIN)/wmkdep $(WMKDEP_FLAGS) -o$@ -I$(*D) $(LIB_HEADER_DIRS) $<
 
 #------------------------------------------------------------------------------

wmake/scripts/AllwmakeParseArguments

@@ -6,20 +6,8 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # File
 #     wmake/scripts/AllwmakeParseArguments

wmake/scripts/cmakeFunctions

@@ -6,20 +6,8 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Script
 #     cmakeFunctions

wmake/scripts/makeDepend

@@ -0,0 +1,62 @@
+#!/bin/sh
+#------------------------------------------------------------------------------
+# =========                 |
+# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+#  \\    /   O peration     |
+#   \\  /    A nd           | Copyright (C) 2018 OpenCFD Ltd.
+#    \\/     M anipulation  |
+#------------------------------------------------------------------------------
+# License
+#     This file is part of OpenFOAM, licensed under GNU General Public License
+#     <http://www.gnu.org/licenses/>.
+#
+# Script
+#     makeDepend
+#
+# Description
+#     Wrapping for cpp -M with argument/options compatible with
+#     <wmake/rules/General/transform> calls of wmkdepend or wmkdep.
+#
+#     This is for testing purposes only, but could used as a hook for
+#     other dependency generation systems (eg, ninja).
+#
+# Note
+#     The order of the calling options is fragile.
+#     Handling of '-eENV' not working.
+#
+# Usage
+#     makeDepend [-eENV...] [-q] [-oFile] ...
+#
+#         -eENV     Environment variable path substitutions.
+#         -oFile    Write output to File.
+#         -q        Suppress 'No such file' warnings.
+#
+#------------------------------------------------------------------------------
+unset envChanges output
+
+while [ "$#" -gt 1 ]
+do
+    case "$1" in
+    -q*)   # quiet - ignore
+    ;;
+
+    -e*)  # -eENV - ignore for now
+        envChanges="$envChanges ${1#-e}"
+        ;;
+
+    -o*)  # -oFILE
+        output="${1#-o}"
+        ;;
+
+    *)
+        break
+        ;;
+    esac
+    shift
+done
+
+cpp -x c++ -std=c++11 -M  \
+    -DWM_"$WM_PRECISION_OPTION" -DWM_LABEL_SIZE="$WM_LABEL_SIZE" \
+    $@ $output
+
+#------------------------------------------------------------------------------

wmake/scripts/makeFiles

@@ -7,20 +7,8 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Script
 #     makeFiles

wmake/scripts/makeOptions

@@ -7,20 +7,8 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Script
 #     makeOptions

wmake/scripts/makeTargetDir

@@ -7,20 +7,8 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Script
 #     makeTargetDir

wmake/scripts/wcleanBuild

@@ -7,20 +7,8 @@
 #    \\/     M anipulation  |
 #-------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Script
 #     wcleanBuild

wmake/scripts/wmakeFunctions

@@ -6,20 +6,8 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
-#     This file is part of OpenFOAM.
+#     This file is part of OpenFOAM, licensed under GNU General Public License
-#
+#     <http://www.gnu.org/licenses/>.
-#     OpenFOAM is free software: you can redistribute it and/or modify it
-#     under the terms of the GNU General Public License as published by
-#     the Free Software Foundation, either version 3 of the License, or
-#     (at your option) any later version.
-#
-#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-#     for more details.
-#
-#     You should have received a copy of the GNU General Public License
-#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Script
 #     wmakeFunctions

wmake/src/Makefile

@@ -86,7 +86,7 @@ $(WMAKE_BIN)/wmkdepend: wmkdepend.cpp
 # $(WMAKE_BIN)/wmkdepend: wmkdepend.rl
 # 	@mkdir -p $(WMAKE_BIN)
 # 	$(call QUIET_MESSAGE,ragel,$(<F))
-# 	$E ragel -o $@.cpp $(<F) && $(CC) $(c++FLAGS) $(c++LESSWARN) $@.cpp -o $@
+# 	$E ragel -G2 -o $@.cpp $(<F) && $(CC) $(c++FLAGS) $(c++LESSWARN) $@.cpp -o $@
 # 	@rm -f $@.cpp 2>/dev/null
 
 #------------------------------------------------------------------------------

wmake/src/wmkdepend.rl

@@ -25,24 +25,32 @@ Application
     wmkdepend
 
 Description
-    A fast dependency list generator that emulates the behaviour and output
+    A fast dependency list generator that emulates the behaviour and
-    of cpp -M. However, the output contains no duplicates and is thus
+    output of cpp -M.
-    approx. 40% faster than cpp.
-    It also handles missing files somewhat more gracefully.
 
-    The algorithm uses a Ragel-generated lexer to scan for includes and
+    The lexer for include statements uses a Ragel-generated lexer and
-    searches the files found.
+    then searches the files found. Each file is visited only once,
-    The files are only visited once (their names are hashed),
+    which helps make this faster than cpp. It also handles missing
-    which helps make this faster than cpp.
+    files somewhat more gracefully.
+
+    The goto-based finite state machine (FSM) is generated with
+
+        ragel -G2 -o wmkdepend.cpp wmkdepend.rl
 
 Usage
     wmkdepend [-Idir..] [-iheader...] [-eENV...] [-oFile] [-q] filename
 
+Note
+    May not capture all possible corner cases or line continuations such as
+
+        #include \
+            "file.H"
+
 \*---------------------------------------------------------------------------*/
 /*
  * With cpp:
  *
- * cpp -x c++ -std=c++11 -nostdinc -nostdinc++
+ * cpp -x c++ -std=c++11
  *     -M -DWM_$(WM_PRECISION_OPTION) -DWM_LABEL_SIZE=$(WM_LABEL_SIZE) |
  * sed -e 's,^$(WM_PROJECT_DIR)/,$$(WM_PROJECT_DIR)/,' \
  *     -e 's,^$(WM_THIRD_PARTY_DIR)/,$$(WM_THIRD_PARTY_DIR)/,'
@@ -57,7 +65,7 @@ Usage
 #include <unordered_set>
 #include <vector>
 
-// Sizing for read buffer
+// Length of the input read buffer
 #define READ_BUFLEN 16384
 
 // The executable name (for messages), without requiring access to argv[]
@@ -78,7 +86,6 @@ void usage()
         "  -eENV     Environment variable path substitutions.\n"
         "  -oFile    Write output to File.\n"
         "  -q        Suppress 'No such file' warnings.\n"
-        "  -v        Some verbosity.\n"
         "\nDependency list generator, similar to 'cpp -M'\n\n";
 }
 
@@ -160,7 +167,7 @@ namespace Files
     //     /openfoam/project/path/directory/xyz
     //  -> $(WM_PROJECT_DIR)/directory/xyz
     //
-    FILE* fopen_file(std::string fileName)
+    FILE* fopen_file(const std::string& fileName)
     {
         const auto len = fileName.size();
         const char *fname = fileName.c_str();
@@ -262,7 +269,7 @@ namespace Files
                 std::cerr << ": " << strerror(errno);
             }
 
-            std::cerr << "\n" << std::flush;
+            std::cerr << '\n' << std::flush;
         }
 
         return filePtr;
@@ -273,37 +280,34 @@ namespace Files
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-// Ragel machine requirements: text start/end, action, code state
+// Ragel machine requirements: token start/end, action, code state
 // are defined later (prior to use)
 
 %%{
-    machine scanInclude;
+    machine wmkdep;
     write   data nofinal;
 
-    action  start_incl { include_start = fpc; }
+    hashInclude = space* '#' space* 'include' space* ; ## include directive
+    includeName = [^\"<>]+ ;                    ## Text between "" or <>
 
-    action  end_incl
+    action  begInclude { ts_inclName = fpc; }   ## Remember start position
+
+    ## No 'if (ts_inclName) ...' guard needed (implicit via the FSM)
+    action  endInclude
     {
-        if (include_start)
+        // std::cerr << std::string(ts, (te-ts)) << '\n';
-        {
+        processFile(std::string(ts_inclName, (fpc - ts_inclName)));
-            processFile(std::string(include_start, (fpc - include_start)));
-        }
-        include_start = nullptr;
     }
 
-    consume_comment :=
+    consume_comment := any* :>> '*/' @{ fgoto main; };
-        any* :>> '*/' @{ fgoto main; };
-
-    user_include =
-        '#' space* 'include' space* '"' %start_incl [^\"]+ %end_incl '"' ;
 
     main := |*
 
-    # Single and double strings
+    # Single and double quoted strings
     ( 'L'? "'" ( [^'\\\n] | /\\./ )* "'") ;     # " swallow
     ( 'L'? '"' ( [^"\\\n] | /\\./ )* '"') ;     # ' swallow
 
-    user_include ;
+    hashInclude '"' %begInclude includeName %endInclude '"' ;
 
     '/*' { fgoto consume_comment; };
     '//' [^\n]* '\n' ;
@@ -323,67 +327,72 @@ void processFile(const std::string& fileName)
     FILE* infile = Files::open(fileName);
     if (!infile) return;
 
-
+    // ts, te  = Ragel token start/end points (required naming)
-    // Ragel text start/end, action, code state
+    // act, cs = Ragel action, code state, respectively (required naming)
-    char *ts = nullptr, *te = nullptr;
+    char *ts, *te;
     unsigned act, cs;
 
-    // For the include action:
+    %% write init ;             ## /* FSM initialization here */;
-    char *include_start = nullptr;
 
-    %%{ write init; }%%
+    // Token start of include filename (for begInclude, endInclude actions)
+    char *ts_inclName = nullptr;
 
     // Buffering
-    char buf[READ_BUFLEN];
+    char inbuf[READ_BUFLEN];
-    size_t bytesPending = 0;
+    size_t pending = 0;
 
-    // Do the first read
+    // Processing loop (as per Ragel pdf example)
     for (bool good = true; good; /*nil*/)
     {
-        const size_t avail = READ_BUFLEN - bytesPending;
+        const size_t avail = READ_BUFLEN - pending;
 
         if (!avail)
         {
             // We overfilled the buffer while trying to scan a token...
             std::cerr
-                << "OUT OF BUFFER SPACE while scanning " << fileName << '\n';
+                << EXENAME ": buffer full while scanning '"
+                << fileName << "'\n";
             break;
         }
 
-        char *p = buf + bytesPending;
+        // p, pe  = Ragel parsing point and parsing end (required naming)
-        const size_t bytesRead = ::fread(p, 1, avail, infile);
+        // eof    = Ragel EOF point (required naming)
 
-        char *pe = p + bytesRead;
+        char *p = inbuf + pending;
+        const size_t gcount = ::fread(p, 1, avail, infile);
+
+        char *pe = p + gcount;
         char *eof = nullptr;
 
-        // If we see eof then append the EOF char.
+        if (!gcount)    // Could also use feof(infile)
-        if (feof(infile))
         {
+            // Tag 'pe' as being the EOF for the FSM as well
             eof = pe;
             good = false;
         }
 
-        %%{ write exec; }%%
+        %% write exec ;         ## /* FSM execution here */;
 
-        if (cs == scanInclude_error)
+        if (cs == wmkdep_error)
         {
-            // Machine failed before finding a token
+            // FSM failed before finding a token
-            std::cerr << "PARSE ERROR while scanning " << fileName << '\n';
+            std::cerr
+                << EXENAME ": parse error while scanning '"
+                << fileName << "'\n";
             break;
         }
 
-        // Now set up the prefix.
+        if (ts)
-        if (ts == nullptr)
         {
-            bytesPending = 0;
+            // Preserve incomplete token
+            pending = pe - ts;
+            ::memmove(inbuf, ts, pending);
+            te = inbuf + (te - ts);   // token end (after memmove)
+            ts = inbuf;               // token start
         }
         else
         {
-            // There are data that needs to be shifted over.
+            pending = 0;
-            bytesPending = pe - ts;
-            ::memmove(buf, ts, bytesPending);
-            te -= (ts-buf);
-            ts = buf;
         }
     }
     fclose(infile);