add lex example

3 files changed, 208 insertions, 0 deletions

diff --git a/examples/lex.c b/examples/lex.c
new file mode 100644
index 0000000..a0f33e7
--- /dev/null
+++ b/examples/lex.c
@@ -0,0 +1,67 @@
+// examples/lex.c
+
+#include <stddef.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define CMMM__RS274NGC__LEXEME__STRIP_VENDOR
+#define CMMM__RS274NGC__LEXER__STRIP_VENDOR
+
+#include "utils.h"
+#include "cmmm/rs274ngc/lexer.h"
+
+int
+main (int argc, char *argv[])
+{
+  const char *filename;
+  struct cmmm__sv rs274ngc;
+  struct cmmm__arena a = {0};
+
+  if (argc == 1) {
+    filename = "-";
+    rs274ngc = read_until_eof (&a, 0);
+  } else if (argc == 2) {
+    filename = argv[1];
+    int fd = open (filename, O_RDONLY);
+    rs274ngc = mmap_whole (fd);
+  } else {
+    exit (EXIT_FAILURE);        // TODO(cmmm): better DIE
+  }
+
+  // printf (SV_FMT, SV_FMTA (rs274ngc));
+  // return 0;
+
+  struct rs274ngc__lexer lexer =
+    rs274ngc__lexer__from_string_view (rs274ngc, filename);
+
+  int line = 1;
+  const char *bol = lexer.lexeme.sv.begin;
+  for (;; rs274ngc__lexer__next_lexeme (&lexer)) {
+    struct rs274ngc__lexeme lexeme = lexer.lexeme;
+
+    if (lexeme.kind == RS274NGC__LEXEME__KIND__END_OF_FILE) {
+      printf ("%s:%d:%ld: %s\n",
+              lexer.filename, line, lexeme.sv.begin - bol,
+              rs274ngc__lexeme__kind_names[lexeme.kind]);
+      break;
+    };
+
+    if (lexeme.kind == RS274NGC__LEXEME__KIND__END_OF_LINE) {
+      printf ("%s:%d:%ld: %s\n",
+              lexer.filename, line, lexeme.sv.begin - bol + 1,
+              rs274ngc__lexeme__kind_names[lexeme.kind]);
+      bol = lexeme.sv.end;
+      line += 1;
+      continue;
+    }
+
+    printf ("%s:%d:%ld-%ld: %s: |"SV_FMT"|\n",
+            lexer.filename, line, lexeme.sv.begin - bol + 1, lexeme.sv.end - bol,
+            rs274ngc__lexeme__kind_names[lexeme.kind], SV_FMTA (lexeme.sv));
+  }
+
+  return EXIT_SUCCESS;
+}
+
+// examples/lex.c ends here
diff --git a/examples/utils.c b/examples/utils.c
new file mode 100644
index 0000000..35c8d98
--- /dev/null
+++ b/examples/utils.c
@@ -0,0 +1,77 @@
+// examples/utils.c
+
+#include <stddef.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "utils.h"
+
+struct cmmm__sv
+mmap_whole (int fd)
+{
+  struct cmmm__sv sv;
+
+  struct stat st;
+  int ret = fstat (fd, &st);
+  UNUSED (ret);
+  ASSERT_MSG (ret != -1, "fstat: %s", strerror (errno));
+  ASSERT (S_ISREG (st.st_mode));
+
+  posix_fadvise (fd, 0, st.st_size, POSIX_FADV_SEQUENTIAL |
+                                    POSIX_FADV_WILLNEED |
+                                    POSIX_FADV_NOREUSE);
+
+  sv.begin = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE |
+                                                MAP_POPULATE |
+                                                MAP_NONBLOCK, fd, 0);
+  ASSERT_MSG (sv.begin != MAP_FAILED, "mmap: %s", strerror (errno));
+  posix_madvise ((void *) sv.begin, st.st_size, MADV_HUGEPAGE | MADV_WILLNEED);
+
+  sv.end = sv.begin + st.st_size;
+
+  return sv;
+}
+
+struct cmmm__sv
+read_until_eof (ARENA arena, int fd)
+{
+  struct arena__checkpoint checkpoint = arena__checkpoint (arena);
+  size_t reserve = 4096;
+
+  size_t read_total = 0;
+  for (;;) {
+    arena__reserve (arena, reserve, 1);
+    ssize_t bytes_read = read (fd,
+                               arena__next_free (arena, 1),
+                               arena__room      (arena, 1));
+
+    if (bytes_read < 0) {
+      DEBUG ("fstat: %s", strerror (errno));
+      arena__rollback_to (arena, checkpoint);
+      return (struct cmmm__sv) {0};
+    }
+
+    if (bytes_read == 0)
+      break;
+
+    read_total += bytes_read;
+
+    arena__allocate_fast (arena, bytes_read, 1);
+    reserve *= 2;
+  }
+
+  if (arena->tail->free == checkpoint.free)
+    return (struct cmmm__sv) {0};
+
+  char *ptr = arena__coalesce_from_fast (arena, checkpoint, read_total);
+  return (struct cmmm__sv) {
+    .begin = ptr,
+    .end   = ptr + read_total,
+  };
+}
+
+// examples/utils.c ends here
diff --git a/examples/utils.h b/examples/utils.h
new file mode 100644
index 0000000..dc48a85
--- /dev/null
+++ b/examples/utils.h
@@ -0,0 +1,64 @@
+// examples/utils.h
+
+#ifndef INCLUDED_UTILS_H
+#define INCLUDED_UTILS_H
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#define CMMM__STRING_VIEW__STRIP_VENDOR
+#define CMMM__STRING_VIEW__SHORT_NAMESPACE
+#define CMMM__ARENA__STRIP_VENDOR
+
+#include "cmmm/string-view.h"
+#include "cmmm/arena.h"
+
+#define NOP() do {} while (0)
+
+#define UNUSED(EXP) ((void) EXP)
+
+#define ARRAY_LENGTH(ARR) (sizeof (ARR)/sizeof (*(ARR)))
+
+#ifndef NDEBUG
+#include <assert.h>
+#define ASSERT assert
+#else
+#define ASSERT(...) NOP ()
+#endif
+
+#define STRINGIFY(ARG) #ARG
+#define TO_STRING(ARG) STRINGIFY (ARG)
+
+#ifndef NDEBUG
+#define DEBUG(FMT, ...)                                                 \
+  fprintf (stderr, __FILE__ ":" TO_STRING (__LINE__) ": %s: " FMT "\n", \
+       __func__ __VA_OPT__ (,) __VA_ARGS__)
+#define LOG(FMT, ...)                                                 \
+  fprintf (stderr, __FILE__ ":" TO_STRING (__LINE__) ": %s: " FMT "\n", \
+       __func__ __VA_OPT__ (,) __VA_ARGS__)
+#else
+#define DEBUG(...) NOP ()
+#define LOG(FMT, ...)                                           \
+  fprintf (stderr, FMT "\n", __VA_OPT__ (,) __VA_ARGS__)
+#endif
+
+#define   ERROR(FMT, ...) LOG ("error: "   FMT __VA_OPT__ (,) __VA_ARGS__)
+#define WARNING(FMT, ...) LOG ("warning: " FMT __VA_OPT__ (,) __VA_ARGS__)
+#define  NOTICE(FMT, ...) LOG ("notice: "  FMT __VA_OPT__ (,) __VA_ARGS__)
+#define    INFO(FMT, ...) LOG ("info: "    FMT __VA_OPT__ (,) __VA_ARGS__)
+
+#ifndef NDEBUG
+#define ASSERT_MSG(EXP, FMT, ...) do if (!(EXP)) {      \
+      DEBUG (FMT __VA_OPT__ (,) __VA_ARGS__);           \
+      exit (EXIT_FAILURE);                              \
+    } while (0)
+#else
+#define ASSERT_MSG(EXP, FMT, ...) NOP ()
+#endif
+
+struct cmmm__sv mmap_whole (int fd);
+struct cmmm__sv read_until_eof (ARENA, int fd);
+
+#endif // INCLUDED_UTILS_H
+
+// examples/utils.c ends here