Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MacOS: code fails to compile under current Xcode #6

Open
mouse07410 opened this issue Apr 6, 2020 · 1 comment
Open

MacOS: code fails to compile under current Xcode #6

mouse07410 opened this issue Apr 6, 2020 · 1 comment

Comments

@mouse07410
Copy link

mouse07410 commented Apr 6, 2020

This happens with the ref/Makefile intact:

$ /usr/bin/gcc -v
Configured with: --prefix=/Applications/Xcode.app/Contents/Developer/usr --with-gxx-include-dir=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/4.2.1
Apple clang version 11.0.3 (clang-1103.0.32.29)
Target: x86_64-apple-darwin19.4.0
Thread model: posix
InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin
$ make
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -o PQCgenKAT_cpakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
rng.c:10:10: fatal error: 'openssl/conf.h' file not found
#include <openssl/conf.h>
         ^~~~~~~~~~~~~~~~
1 error generated.
make: *** [PQCgenKAT_cpakem512] Error 1

After pointing the compiler at the correct location of the OpenSSL-1.1.1 (there's no other usable OpenSSL on this box, except for OpenSSL-3.0 Dev master):

diff --git a/ref/Makefile b/ref/Makefile
index f7d4469..5f6fbba 100644
--- a/ref/Makefile
+++ b/ref/Makefile
@@ -1,6 +1,6 @@
 CC = /usr/bin/gcc
-CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib
+NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib
 
 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c

the code in ref/ builds but crashes:

$ make
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c test_newhope.c -o test_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c test_newhope.c -o test_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c test_newhope.c -o test_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c test_newhope.c -o test_ccakem1024
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem1024
$ ./PQCgenKAT_ccakem512 
Segmentation fault: **11**

The problem appears to be stack alignment that gets screwed up by the Xcode compilers - from the crash report:

Thread 0 Crashed:: Dispatch queue: com.apple.main-thread
0   libdyld.dylib                 	0x00007fff6c5d970a stack_not_16_byte_aligned_error + 0
1   ???                           	0x00007ffee97bce08 0 + 140732815625736

This workaround (adding -ffrestanding flag) helps:

diff --git a/ref/Makefile b/ref/Makefile
index f7d4469..f7fe9bc 100644
--- a/ref/Makefile
+++ b/ref/Makefile
@@ -1,6 +1,6 @@
 CC = /usr/bin/gcc
-CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CFLAGS = -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib
+NISTFLAGS = -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib
 
 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c

and produces desirable results:

$ make
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c test_newhope.c -o test_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c test_newhope.c -o test_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c test_newhope.c -o test_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c test_newhope.c -o test_ccakem1024
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem1024
$ ./PQCgenKAT_ccakem512 
$ 

Switching from Xcode clang to Macports-installed clang-mp-9.0 alleviated the need to add -ffreestanding flag.

Update

Similar problem with avx2/ code, for the same reason ("stack not 16-byte aligned").
Compiling with Macports-installed clang-mp-9.0, or adding -ffreestanding flag resolved it here as well.

@mouse07410 mouse07410 changed the title MacOS: ref fails to compile under current Xcode MacOS: code fails to compile under current Xcode Apr 6, 2020
@mouse07410
Copy link
Author

Finally, the following patch allows build and tests succeed using Xcode-11.4 clang:

diff --git a/avx2/Makefile b/avx2/Makefile
index 83c0285..41ef314 100644
--- a/avx2/Makefile
+++ b/avx2/Makefile
@@ -1,6 +1,6 @@
-CC = /usr/bin/gcc
-CFLAGS = -no-pie -Wall -Wextra -g -O3 -fomit-frame-pointer -msse2avx -mavx2 -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CC ?= gcc
+CFLAGS = -ffreestanding -fno-stack-check -no-pie -Wall -Wextra -g -O3 -fomit-frame-pointer -msse2 -mavx -mavx2 -march=native -I/opt/local/include -L/opt/local/lib 
+NISTFLAGS = -ffreestanding -fno-stack-check -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib
 
 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c  verify.c cpapke.c ntt_double.s ntt.c precomp.c
@@ -19,7 +19,7 @@ all:	PQCgenKAT_cpakem512 \
 			speed_cpakem512 \
 			speed_cpakem1024 \
 			speed_ccakem512 \
-			speed_ccakem1024 \
+			speed_ccakem1024
 
 PQCgenKAT_cpakem512: $(SOURCES) $(HEADERS) PQCgenKAT_kem.c rng.c rng.h
 	ln -sf cpakem.h api.h
diff --git a/avx2/poly.h b/avx2/poly.h
index 1bfa840..92d4d0a 100644
--- a/avx2/poly.h
+++ b/avx2/poly.h
@@ -2,6 +2,7 @@
 #define POLY_H

 #include <stdint.h>
+#include <stddef.h>
 #include "params.h"
 
 typedef struct {
diff --git a/avx2/speed.c b/avx2/speed.c
index bb9c225..a3f0d64 100644
--- a/avx2/speed.c
+++ b/avx2/speed.c
@@ -13,7 +13,7 @@ static int cmp_llu(const void *a, const void*b)
   return 0;
 }
 
-static unsigned long long median(unsigned long long *l, size_t llen)
+static unsigned long long median(unsigned long long *l, const size_t llen)
 {
   qsort(l,llen,sizeof(unsigned long long),cmp_llu);
 
@@ -21,7 +21,7 @@ static unsigned long long median(unsigned long long *l, size_t llen)
   else return (l[llen/2-1]+l[llen/2])/2;
 }
 
-static unsigned long long average(unsigned long long *t, size_t tlen)
+static unsigned long long average(unsigned long long *t, const size_t tlen)
 {
   unsigned long long acc=0;
   size_t i;
@@ -30,7 +30,7 @@ static unsigned long long average(unsigned long long *t, size_t tlen)
   return acc/(tlen);
 }
 
-static void print_results(const char *s, unsigned long long *t, size_t tlen)
+static void print_results(const char *s, unsigned long long *t, const size_t tlen)
 {
   size_t i;
   printf("%s", s);
diff --git a/avx2/test_newhope.c b/avx2/test_newhope.c
index e30ddcd..23b85d1 100644
--- a/avx2/test_newhope.c
+++ b/avx2/test_newhope.c
@@ -1,9 +1,9 @@
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
 #include "api.h"
 #include "poly.h"
 #include "randombytes.h"
-#include <math.h>
-#include <stdio.h>
-#include <string.h>
 
 #define NTESTS 1000
 
@@ -24,7 +24,7 @@ int test_keys()
     //Bob derives a secret key and creates a response
     crypto_kem_enc(sendb, key_b, pk);
   
-    //Alice uses Bobs response to get her secre key
+    //Alice uses Bobs response to get her secret key
     crypto_kem_dec(key_a, sendb, sk_a);
 
     if(memcmp(key_a, key_b, 32))
@@ -65,7 +65,7 @@ int test_invalid_sk_a()
     //Replace secret key with random values
     fread(sk_a, CRYPTO_SECRETKEYBYTES, 1, urandom); 
   
-    //Alice uses Bobs response to get her secre key
+    //Alice uses Bobs response to get her secret key
     crypto_kem_dec(key_a, sendb, sk_a);
 
     if(!memcmp(key_a, key_b, 32))
@@ -99,7 +99,7 @@ int test_invalid_ciphertext()
     //Change some byte in the ciphertext (i.e., encapsulated key)
     sendb[42] ^= 23;
   
-    //Alice uses Bobs response to get her secre key
+    //Alice uses Bobs response to get her secret key
     crypto_kem_dec(key_a, sendb, sk_a);
 
     if(!memcmp(key_a, key_b, 32))
diff --git a/avx2/verify.c b/avx2/verify.c
index 7d01a47..b634a24 100644
--- a/avx2/verify.c
+++ b/avx2/verify.c
@@ -2,7 +2,7 @@
 #include <stdint.h>
 
 /* returns 0 for equal strings, 1 for non-equal strings */
-int verify(const unsigned char *a, const unsigned char *b, size_t len)
+int verify(const unsigned char *a, const unsigned char *b, const size_t len)
 {
   uint64_t r;
   size_t i;
diff --git a/avx2/verify.h b/avx2/verify.h
index 5394316..693af4b 100644
--- a/avx2/verify.h
+++ b/avx2/verify.h
@@ -4,7 +4,7 @@
 #include <stdio.h>
 
 /* returns 0 for equal strings, 1 for non-equal strings */
-int verify(const unsigned char *a, const unsigned char *b, size_t len);
+int verify(const unsigned char *a, const unsigned char *b, const size_t len);
 
 /* b = 1 means mov, b = 0 means don't mov*/
 void cmov(unsigned char *r, const unsigned char *x, size_t len, unsigned char b);
diff --git a/ref/Makefile b/ref/Makefile
index f7d4469..a133512 100644
--- a/ref/Makefile
+++ b/ref/Makefile
@@ -1,6 +1,6 @@
-CC = /usr/bin/gcc
-CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CC ?= gcc
+CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib
+NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib
 
 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c
@@ -18,7 +18,7 @@ all:	PQCgenKAT_cpakem512 \
 			speed_cpakem512 \
 			speed_cpakem1024 \
 			speed_ccakem512 \
-			speed_ccakem1024 \
+			speed_ccakem1024 
 
 PQCgenKAT_cpakem512: $(SOURCES) $(HEADERS) PQCgenKAT_kem.c rng.c rng.h
 	ln -sf cpakem.h api.h
diff --git a/ref/poly.h b/ref/poly.h
index efe5f49..4728a30 100644
--- a/ref/poly.h
+++ b/ref/poly.h
@@ -2,6 +2,7 @@
 #define POLY_H
 
 #include <stdint.h>
+#include <stddef.h>
 #include "params.h"

 /* 

I took the liberty of adding const where it belonged and the required header files where they were missing. Also, re-ordered the headers.

Here's the patch file: macosx.patch.zip

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

1 participant