Bug 571722: Add ALL_FLAGS for scanner discovery

This change adds the ALL_FLAGS that does not limit tool options to
those declared as IOption::isForScannerDiscovery when launching the
compiler to discover compiler built-ins.

This is needed as many other flags, either entered manually in "Other
flags" or some of the existing flags with checkboxes such as "-ansi",
"-fPIC", and "-fstack-protector-all" which all affect scanner discovery
as they can all change what macros are built-in to the compiler.

The current solution has as a drawback that some settings, like -I and -D
then appear twice. For example in the "Includes" node in the "Project
Explorer"

My only reservation about this change is if there is an option
that can be specified successfully at build time, but when used
at scanner discovery time causes the compiler to fail, or return
incorrect results. Therefore I have added a new field,
excludeFromScannerDiscovery to tool options (buildDefinitions
extension point) that allows tool integrators to always exclude
a command line option from ALL_FLAGS. I have also added
a new "Other flags (excluded from discovery)" to the
"Miscellaneous" tab to allow compiler options to be entered
by the user.

NewAndNoteworthy/CDT-11.0.md

@@ -34,6 +34,28 @@ Please see the corresponding issue for more details.
   - `org.eclipse.cdt.lsp.cquery`
   - `org.eclipse.cdt.lsp.ui`
 
+# Build
+
+## Scanner Discovery able to consider all flags
+
+By default CDT only considers some other compile flags when computing the compiler built-in includes and macros.
+To have CDT consider all compiler flags when computing the compiler built-in includes and macros the `${ALL_FLAGS}` can be specified in  _Command to get compiler specs:_.
+
+For example, when `-ansi` is passed to GCC, the `__STRICT_ANSI__` macros is defined.
+For CDT to know that, the `-ansi` needs to be passed to GCC when collecting the compiler built-ins, sometimes in CDT called "scanning" or "compiler specs".
+If you code with, for example, `#ifdef __STRICT_ANSI__` it will be highlighted correctly only if CDT knows about it.
+Therefore in this situation it may be best to use this new feature.
+
+To enable using all flags when calculating built-ins, change the default `${FLAGS}` to `${ALL_FLAGS}` in _Project Properties_ -\> _C/C++ General_ -\> _Preprocessor Include Paths, Macros etc._ -\> _Providers_ tab -\> _CDT GCC Built-in Compiler Settings_ -\> _Command to get compiler specs:_.
+
+<p align="center"><img src="images/CDT-11.0-all-flags.png" width="50%"></p>
+
+If your compiler has a command line option that interferes with scanner discovery, it can be declared as such in the `plugin.xml` (see below) or entered into the _Other flags (excluded from discovery)_ option in the _Miscellaneous_ of build settings.
+
+See the online help sections on [scanner discovery](https://help.eclipse.org/latest/topic/org.eclipse.cdt.doc.user/concepts/cdt_c_scanner_discovery.htm) and [scanner discovery preferences](https://help.eclipse.org/latest/topic/org.eclipse.cdt.doc.user/reference/cdt_u_pref_build_scanner_discovery.htm) for more information on scanner discovery.
+
+See https://github.com/eclipse-cdt/cdt/pull/158.
+
 # Debug
 
 ## C/C++ Dynamic Printf Breakpoints
@@ -82,6 +104,13 @@ The `ICBuildConfiguration` and `IToolChain` interfaces now have a method, `getBi
 
 See https://github.com/eclipse-cdt/cdt/pull/75
 
+## Exclude from scanner discovery flag for options
+
+`IOption` has a new flag called `isExcludedFromScannerDiscovery`.
+This flag can be set on options to exclude them from scanner discovery when using the new `${ALL_FLAGS}` variable.
+
+See https://github.com/eclipse-cdt/cdt/pull/158.
+
 # Bugs Fixed in this Release
 
 See [GitHub milestones](https://github.com/eclipse-cdt/cdt/milestone/2?closed=1) and for bugs that haven't been transitioned to GitHub please see Bugzilla report [Bugs Fixed in CDT 11.0](https://bugs.eclipse.org/bugs/buglist.cgi?bug_status=RESOLVED&bug_status=VERIFIED&bug_status=CLOSED&classification=Tools&product=CDT&query_format=advanced&resolution=FIXED&target_milestone=11.0.0).

build/org.eclipse.cdt.managedbuilder.core.tests/plugin.xml

@@ -9601,6 +9601,21 @@
                   resourceFilter="all"
                   valueType="string">
             </option>
+            <option
+                  command="-not-excluded-from-scanner-discovery"
+                  id="cdt.managedbuilder.lsp.tests.option.not-excluded-sd"
+                  isAbstract="false"
+                  resourceFilter="all"
+                  valueType="string">
+            </option>
+            <option
+                  command="-is-excluded-from-scanner-discovery"
+                  id="cdt.managedbuilder.lsp.tests.option.is-excluded-sd"
+                  isAbstract="false"
+                  resourceFilter="all"
+                  excludeFromScannerDiscovery="true"
+                  valueType="string">
+            </option>
             <option
                   command="-bool-option"
                   defaultValue="true"

build/org.eclipse.cdt.managedbuilder.core.tests/tests/org/eclipse/cdt/managedbuilder/core/tests/ManagedBuildCoreTests.java

@@ -34,9 +34,6 @@
 import org.eclipse.core.runtime.IConfigurationElement;
 import org.junit.Assert;
 
-import junit.framework.Test;
-import junit.framework.TestSuite;
-
 public class ManagedBuildCoreTests extends BaseTestCase {
 	private static IProjectType exeType;
 	private static IProjectType libType;
@@ -46,14 +43,6 @@ public ManagedBuildCoreTests(String name) {
 		super(name);
 	}
 
-	public static Test suite() {
-		TestSuite suite = new TestSuite(ManagedBuildCoreTests.class.getName());
-		suite.addTest(new ManagedBuildCoreTests("testLoadManifest"));
-		suite.addTest(new ManagedBuildCoreTests("testTreeOptions"));
-		suite.addTest(new ManagedBuildCoreTests("testOptionsAttributeUseByScannerDiscovery"));
-		return suite;
-	}
-
 	/**
 	 * Navigates through a CDT 2.1 manifest file and verifies that the
 	 * definitions are loaded correctly.
@@ -673,4 +662,19 @@ public void testOptionsAttributeUseByScannerDiscovery() throws Exception {
 		assertNotNull(option);
 		assertEquals(true, option.isForScannerDiscovery());
 	}
+
+	/**
+	 * Tests attribute excludedFromScannerDiscovery.
+	 * @throws Exception
+	 */
+	public void testExcludedFromScannerDiscovery() throws Exception {
+		IOption optionNotExcludedFromSD = ManagedBuildManager
+				.getExtensionOption("cdt.managedbuilder.lsp.tests.option.not-excluded-sd");
+		assertNotNull(optionNotExcludedFromSD);
+		assertEquals(false, optionNotExcludedFromSD.isExcludedFromScannerDiscovery());
+
+		IOption option = ManagedBuildManager.getExtensionOption("cdt.managedbuilder.lsp.tests.option.is-excluded-sd");
+		assertNotNull(option);
+		assertEquals(true, option.isExcludedFromScannerDiscovery());
+	}
 } // end class

build/org.eclipse.cdt.managedbuilder.core/schema/buildDefinitions.exsd

@@ -1410,7 +1410,14 @@ Additional special types exist to flag options of special relevance to the build
          <attribute name="useByScannerDiscovery" type="boolean">
             <annotation>
                <documentation>
-                  An optional field to allow the option additionally to be passed to scanner discovery of built-in compiler macros and paths. The default is false.
+                  An optional field to allow the option additionally to be passed to scanner discovery of built-in compiler macros and paths in the ${FLAGS} variable. The default is false.
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="excludeFromScannerDiscovery" type="boolean">
+            <annotation>
+               <documentation>
+                  An optional field to allow the option to be excluded from scanner discovery's ${ALL_FLAGS} variable. The default is false.
                </documentation>
             </annotation>
          </attribute>

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/core/IOption.java

@@ -106,6 +106,8 @@ public interface IOption extends IBuildObject {
 	public static final String COMMAND_FALSE = "commandFalse"; //$NON-NLS-1$
 	/** @since 8.3 */
 	public static final String USE_BY_SCANNER_DISCOVERY = "useByScannerDiscovery"; //$NON-NLS-1$
+	/** @since 9.5 */
+	public static final String EXCLUDE_FROM_SCANNER_DISCOVERY = "excludeFromScannerDiscovery"; //$NON-NLS-1$
 	/** @since 8.0 */
 	public static final String COMMAND_GENERATOR = "commandGenerator"; //$NON-NLS-1$
 	public static final String TOOL_TIP = "tip"; //$NON-NLS-1$
@@ -630,6 +632,15 @@ public interface IOption extends IBuildObject {
 	 */
 	public boolean isForScannerDiscovery();
 
+	/**
+	 * Flag to indicate whether the option should be excluded from scanner discovery.
+	 * @return {@code true} if the option should never be passed to scanner discovery command
+	 *    or {@code false} otherwise.
+	 *
+	 * @since 9.5
+	 */
+	public boolean isExcludedFromScannerDiscovery();
+
 	/**
 	 * Returns the tree root of this option if it is of type {@link #TREE}
 	 * @return tree root of this option or <code>null</code> if not found.

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/internal/core/Option.java

@@ -81,6 +81,7 @@ public class Option extends BuildObject implements IOption, IBuildPropertiesRest
 	private IOptionCommandGenerator commandGenerator;
 	private String commandFalse;
 	private Boolean isForScannerDiscovery;
+	private Boolean isExcludedFromScannerDiscovery;
 	private String tip;
 	private String contextId;
 	private List<String> applicableValuesList;
@@ -224,6 +225,9 @@ public Option(IHoldsOptions parent, String Id, String name, Option option) {
 		if (option.isForScannerDiscovery != null) {
 			isForScannerDiscovery = option.isForScannerDiscovery;
 		}
+		if (option.isExcludedFromScannerDiscovery != null) {
+			isExcludedFromScannerDiscovery = option.isExcludedFromScannerDiscovery;
+		}
 		if (option.tip != null) {
 			tip = option.tip;
 		}
@@ -405,6 +409,12 @@ protected void loadFromManifest(IManagedConfigElement element) {
 			isForScannerDiscovery = Boolean.parseBoolean(isForSD);
 		}
 
+		// isNotForScannerDiscovery
+		String isExcludeFromSD = element.getAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY);
+		if (isExcludeFromSD != null) {
+			isExcludedFromScannerDiscovery = Boolean.parseBoolean(isExcludeFromSD);
+		}
+
 		// Get the tooltip for the option
 		tip = SafeStringInterner.safeIntern(element.getAttribute(TOOL_TIP));
 
@@ -578,6 +588,14 @@ protected void loadFromProject(ICStorageElement element) {
 			}
 		}
 
+		// isNotForScannerDiscovery
+		if (element.getAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY) != null) {
+			String isExcludeFromSD = element.getAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY);
+			if (isExcludeFromSD != null) {
+				isForScannerDiscovery = Boolean.parseBoolean(isExcludeFromSD);
+			}
+		}
+
 		// Get the tooltip for the option
 		if (element.getAttribute(TOOL_TIP) != null) {
 			tip = SafeStringInterner.safeIntern(element.getAttribute(TOOL_TIP));
@@ -887,6 +905,10 @@ public void serialize(ICStorageElement element) throws BuildException {
 			element.setAttribute(USE_BY_SCANNER_DISCOVERY, isForScannerDiscovery.toString());
 		}
 
+		if (isExcludedFromScannerDiscovery != null) {
+			element.setAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY, isExcludedFromScannerDiscovery.toString());
+		}
+
 		if (tip != null) {
 			element.setAttribute(TOOL_TIP, tip);
 		}
@@ -1381,6 +1403,14 @@ public boolean isForScannerDiscovery() {
 		return isForScannerDiscovery;
 	}
 
+	@Override
+	public boolean isExcludedFromScannerDiscovery() {
+		if (isExcludedFromScannerDiscovery == null) {
+			isExcludedFromScannerDiscovery = superClass != null && superClass.isExcludedFromScannerDiscovery();
+		}
+		return isExcludedFromScannerDiscovery;
+	}
+
 	@Override
 	public String getToolTip() {
 		if (tip == null) {

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/internal/core/OptionReference.java

@@ -926,6 +926,11 @@ public boolean isForScannerDiscovery() {
 		return option.isForScannerDiscovery();
 	}
 
+	@Override
+	public boolean isExcludedFromScannerDiscovery() {
+		return option.isExcludedFromScannerDiscovery();
+	}
+
 	@Override
 	public ITreeRoot getTreeRoot() {
 		if (!resolved) {

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/language/settings/providers/AbstractBuiltinSpecsDetector.java

@@ -53,6 +53,7 @@
 import org.eclipse.cdt.internal.core.BuildRunnerHelper;
 import org.eclipse.cdt.internal.core.XmlUtil;
 import org.eclipse.cdt.internal.core.envvar.EnvironmentVariableManager;
+import org.eclipse.cdt.managedbuilder.core.IOption;
 import org.eclipse.cdt.managedbuilder.core.ManagedBuilderCorePlugin;
 import org.eclipse.cdt.managedbuilder.internal.core.ManagedMakeMessages;
 import org.eclipse.cdt.utils.CommandLineUtil;
@@ -101,6 +102,8 @@ public abstract class AbstractBuiltinSpecsDetector extends AbstractLanguageSetti
 	protected static final String COMPILER_MACRO = "${COMMAND}"; //$NON-NLS-1$
 	/** @since 8.3 */
 	protected static final String FLAGS_MACRO = "${FLAGS}"; //$NON-NLS-1$
+	/**	@since 9.5*/
+	protected static final String ALL_FLAGS_MACRO = "${ALL_FLAGS}"; //$NON-NLS-1$
 	protected static final String SPEC_FILE_MACRO = "${INPUTS}"; //$NON-NLS-1$
 	protected static final String SPEC_EXT_MACRO = "${EXT}"; //$NON-NLS-1$
 	protected static final String SPEC_FILE_BASE = "spec"; //$NON-NLS-1$
@@ -279,6 +282,7 @@ public void shutdown() {
 	 *    Normally would come from the tool-chain.<br>
 	 * <b>${INPUTS}</b> - path to spec file which will be placed in workspace area.<br>
 	 * <b>${EXT}</b> - file extension calculated from language ID.
+	 * <b>${ALL_FLAGS}</b> - all the command line flags.
 	 * </ul>
 	 * The parameter could be taken from the extension
 	 * in {@code plugin.xml} or from property file.
@@ -334,6 +338,11 @@ protected String resolveCommand(String languageId) throws CoreException {
 				if (flags != null)
 					cmd = cmd.replace(FLAGS_MACRO, flags);
 			}
+			if (cmd.contains(ALL_FLAGS_MACRO)) {
+				String flags = getAllToolOptions(languageId);
+				if (flags != null)
+					cmd = cmd.replace(ALL_FLAGS_MACRO, flags);
+			}
 			if (cmd.contains(SPEC_FILE_MACRO)) {
 				String specFileName = getSpecFile(languageId);
 				if (specFileName != null)
@@ -933,6 +942,9 @@ protected Optional<String> selectBestSpecFileExtension(List<String> extensions)
 	 * Determine additional options to pass to scanner discovery command.
 	 * These options are intended to come from the tool-chain.
 	 *
+	 * Unlike {@link #getAllToolOptions(String)} this method filters to only
+	 * return a subset of all options. See {@link IOption#isForScannerDiscovery()}
+	 *
 	 * @param languageId - language ID.
 	 * @return additional options to pass to scanner discovery command.
 	 *
@@ -942,6 +954,23 @@ protected String getToolOptions(String languageId) {
 		return ""; //$NON-NLS-1$
 	}
 
+	/**
+	 * Determine additional options to pass to scanner discovery command.
+	 * These options are intended to come from the tool-chain.
+	 *
+	 * Unlike {@link #getToolOptions(String)} this returns all options,
+	 * except for those specifically excluded from scanner discovery
+	 * with {@link IOption#isExcludedFromScannerDiscovery()}
+	 *
+	 * @param languageId - language ID.
+	 * @return additional options to pass to scanner discovery command.
+	 *
+	 * @since 9.5
+	 */
+	protected String getAllToolOptions(String languageId) {
+		return ""; //$NON-NLS-1$
+	}
+
 	@Override
 	public Element serializeAttributes(Element parentElement) {
 		Element elementProvider = super.serializeAttributes(parentElement);

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/language/settings/providers/ToolchainBuiltinSpecsDetector.java

@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Predicate;
 
 import org.eclipse.cdt.core.envvar.IEnvironmentVariable;
 import org.eclipse.cdt.managedbuilder.core.BuildException;
@@ -153,16 +154,18 @@ protected String getSpecFileExtension(String languageId) {
 		return extension.get();
 	}
 
-	@Override
-	protected String getToolOptions(String languageId) {
+	/**
+	 * @since 9.5
+	 */
+	protected String getToolOptions(String languageId, Predicate<IOption> predicate) {
 		Optional<IOption[]> found = languageTool(languageId).flatMap(t -> Optional.of(t.getOptions()));
 		if (!found.isPresent()) {
 			return ""; //$NON-NLS-1$
 		}
 		StringBuilder flags = new StringBuilder();
 		IOption[] options = found.get();
 		for (IOption option : options) {
-			if (option.isForScannerDiscovery()) {
+			if (predicate.test(option)) {
 				try {
 					String optionValue = null;
 					switch (option.getBasicValueType()) {
@@ -207,6 +210,16 @@ protected String getToolOptions(String languageId) {
 		return flags.toString().trim();
 	}
 
+	@Override
+	protected String getToolOptions(String languageId) {
+		return getToolOptions(languageId, IOption::isForScannerDiscovery);
+	}
+
+	@Override
+	protected String getAllToolOptions(String languageId) {
+		return getToolOptions(languageId, Predicate.not(IOption::isExcludedFromScannerDiscovery));
+	}
+
 	@Override
 	protected List<IEnvironmentVariable> getEnvironmentVariables() {
 		if (envMngr == null && currentCfgDescription == null) {

build/org.eclipse.cdt.managedbuilder.gnu.ui/plugin.properties

@@ -188,6 +188,8 @@ Option.Posix.Warn.wwritestring=Treat strings always as const (-Wwrite-strings)
 
 Option.Posix.Verbose=Verbose (-v)
 Option.OtherFlags=Other flags
+Option.OtherFlagsExcludedFromScannerDiscovery=Other flags (excluded from discovery)
+Option.OtherFlagsExcludedFromScannerDiscoveryTip=Flags that will not be passed to the compiler when discovering compiler built-in macros and include paths. Include compiler flags here that may interfere with the correct operation of scanner discovery.
 Option.Posix.Ansi=Support ANSI programs (-ansi)
 Option.PIC=Position Independent Code (-fPIC)
 Option.codecov=Generate gcov information (-ftest-coverage -fprofile-arcs)

build/org.eclipse.cdt.managedbuilder.gnu.ui/plugin.xml

@@ -1440,6 +1440,15 @@
 			  id="gnu.c.compiler.option.misc.other"
 			  valueType="string">
 		  </option>
+		  <option
+			  defaultValue=""
+			  name="%Option.OtherFlagsExcludedFromScannerDiscovery"
+			  tip="%Option.OtherFlagsExcludedFromScannerDiscoveryTip"
+			  category="gnu.c.compiler.category.other"
+			  id="gnu.c.compiler.option.misc.otherExcludedFromScannerDiscovery"
+			  excludeFromScannerDiscovery="true"
+			  valueType="string">
+		  </option>
 		  <option
 			  defaultValue="false"
 			  name="%Option.Posix.Verbose"
@@ -2036,6 +2045,15 @@
 			  id="gnu.cpp.compiler.option.other.other"
 			  valueType="string">
 		  </option>
+		  <option
+			  defaultValue=""
+			  name="%Option.OtherFlagsExcludedFromScannerDiscovery"
+			  tip="%Option.OtherFlagsExcludedFromScannerDiscoveryTip"
+			  category="gnu.cpp.compiler.category.other"
+			  id="gnu.cpp.compiler.option.other.otherExcludedFromScannerDiscovery"
+			  excludeFromScannerDiscovery="true"
+			  valueType="string">
+		  </option>
 		  <option
 			  defaultValue="false"
 			  name="%Option.Posix.Verbose"