language detection

package.json

@@ -3,34 +3,33 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
-    "@monaco-editor/react": "^4.5.1",
+    "@monaco-editor/react": "^4.6.0",
-    "copy-to-clipboard": "^3.3.1",
+    "copy-to-clipboard": "^3.3.3",
-    "history": "^5.0.0",
+    "history": "^5.3.0",
     "local-storage": "^2.0.0",
     "monaco-themes": "^0.4.4",
-    "pako": "^2.0.3",
+    "pako": "^2.1.0",
     "react": "^18.2.0",
-    "react-device-detect": "^2.1.2",
+    "react-device-detect": "^2.2.3",
     "react-dom": "^18.2.0",
     "react-scripts": "^5.0.1",
-    "styled-components": "^5.2.1",
+    "styled-components": "^6.1.1",
-    "typescript": "^4.8.4",
+    "typescript": "^4.4.2",
     "whatwg-mimetype": "^3.0.0"
   },
   "devDependencies": {
-    "@testing-library/jest-dom": "^5.11.4",
+    "@testing-library/jest-dom": "^5.14.1",
-    "@testing-library/react": "^13.3.0",
+    "@testing-library/react": "^13.0.0",
-    "@testing-library/user-event": "^14.2.0",
+    "@testing-library/user-event": "^13.2.1",
-    "@types/jest": "^29.2.2",
+    "@types/jest": "^27.0.1",
     "@types/node": "^18.11.9",
-    "@types/pako": "^2.0.0",
+    "@types/pako": "^2.0.3",
-    "@types/react": "^18.0.25",
+    "@types/react": "^18.2.0",
-    "@types/react-dom": "^18.0.8",
+    "@types/react-dom": "^18.2.0",
-    "@types/styled-components": "^5.1.26",
+    "@types/whatwg-mimetype": "^3.0.2",
-    "@types/whatwg-mimetype": "^3.0.0",
+    "monaco-editor": "^0.44.0",
-    "monaco-editor": "^0.34.1",
+    "prettier": "^3.1.0",
-    "prettier": "^2.7.1",
+    "prettier-plugin-organize-imports": "^3.2.4"
-    "prettier-plugin-organize-imports": "^3.1.1"
   },
   "scripts": {
     "start": "react-scripts start",

src/App.tsx

@@ -1,6 +1,8 @@
 import { useEffect, useState } from 'react';
 import Editor from './components/Editor';
 import { loadFromBytebin } from './util/storage';
+import { Language } from './util/language';
+import { detectLanguage } from './util/detect-language';
 
 const INITIAL = Symbol();
 const LOADING = Symbol();
@@ -13,7 +15,7 @@ export default function App() {
   const [state, setState] = useState<LoadingState>(INITIAL);
   const [forcedContent, setForcedContent] = useState<string>('');
   const [actualContent, setActualContent] = useState<string>('');
-  const [contentType, setContentType] = useState<string>();
+  const [contentType, setContentType] = useState<Language>();
 
   function setContent(content: string) {
     setActualContent(content);
@@ -28,8 +30,14 @@ export default function App() {
       loadFromBytebin(pasteId).then(({ ok, content, type }) => {
         if (ok) {
           setContent(content);
-          if (type) {
+          if (type !== 'plain') {
             setContentType(type);
+          } else {
+            detectLanguage(pasteId).then(detectedLanguage => {
+              if (detectedLanguage) {
+                setContentType(detectedLanguage);
+              }
+            });
           }
         } else {
           setContent(get404Message(pasteId));

src/components/Button.tsx

@@ -9,7 +9,7 @@ const Button = styled.div`
   color: inherit;
   text-decoration: none;
 
-  :hover {
+  &:hover {
     background: ${props => props.theme.highlight};
   }
 

src/components/Editor.tsx

@@ -7,12 +7,13 @@ import themes, { Themes } from '../style/themes';
 import EditorControls from './EditorControls';
 import EditorGlobalStyle from './EditorGlobalStyle';
 import EditorTextArea from './EditorTextArea';
+import { Language } from '../util/language';
 
 export interface EditorProps {
   forcedContent: string;
   actualContent: string;
   setActualContent: (value: string) => void;
-  contentType?: string;
+  contentType?: Language;
   pasteId?: string;
 }
 

src/components/EditorControls.tsx

@@ -4,7 +4,7 @@ import { MutableRefObject, useCallback, useEffect, useState } from 'react';
 import styled from 'styled-components';
 
 import themes, { Themes } from '../style/themes';
-import { languages } from '../util/highlighting';
+import { languages, unknownLanguage } from '../util/language';
 import { saveToBytebin } from '../util/storage';
 import Button from './Button';
 import { ResetFunction } from './Editor';
@@ -104,9 +104,9 @@ export default function EditorControls({
         </Button>
         <MenuButton
           label="language"
-          value={language}
+          value={language === unknownLanguage ? '?' : language}
           setValue={setLanguage}
-          ids={languages}
+          ids={languages as unknown as Record<string, string[]>}
         />
         {readOnly && <Button onClick={unsetReadOnly}>[edit]</Button>}
       </Section>

src/components/EditorGlobalStyle.tsx

@@ -1,7 +1,6 @@
-import { createGlobalStyle, ThemeProps } from 'styled-components';
+import { createGlobalStyle } from 'styled-components';
-import { Theme } from '../style/themes';
 
-const EditorGlobalStyle = createGlobalStyle<ThemeProps<Theme>>`
+const EditorGlobalStyle = createGlobalStyle`
   html, body {
     color-scheme: ${props => props.theme.lightOrDark};
     scrollbar-color: ${props => props.theme.lightOrDark};

src/components/EditorTextArea.tsx

@@ -17,6 +17,7 @@ import themes, { Theme } from '../style/themes';
 
 import type { editor } from 'monaco-editor';
 import { ResetFunction } from './Editor';
+import { logLanguage } from '../util/log-language';
 
 export interface EditorTextAreaProps {
   forcedContent: string;
@@ -60,6 +61,9 @@ export default function EditorTextArea({
       monaco.editor.defineTheme(theme.id, theme.editor);
     }
 
+    monaco.languages.register({ id: 'log' });
+    monaco.languages.setMonarchTokensProvider('log', logLanguage);
+
     monaco.languages.typescript.typescriptDefaults.setDiagnosticsOptions({
       noSemanticValidation: true,
       noSyntaxValidation: true,

src/components/MenuButton.tsx

@@ -92,7 +92,7 @@ const Menu = styled.ul`
   }
 
   > li.selected {
-    ::before {
+    &::before {
       content: '*';
       font-weight: bold;
     }

src/index.tsx

@@ -2,6 +2,7 @@ import React from 'react';
 import ReactDOM from 'react-dom/client';
 import App from './App';
 import './style/base.css';
+import type {} from './style/styled';
 
 const root = ReactDOM.createRoot(
   document.getElementById('root') as HTMLElement

src/style/styled.d.ts

@@ -0,0 +1,6 @@
+import 'styled-components';
+import { Theme } from './themes';
+
+declare module 'styled-components' {
+  export interface DefaultTheme extends Theme {}
+}

src/style/themes.ts

@@ -59,6 +59,11 @@ const themes: Themes = {
         keyword: '#ff7b72',
         type: '#ffa657',
         variable: '#ffa657',
+        logInfo: '#3fb950', // green.3
+        logError: '#f85149', // red.4
+        logWarning: '#d29922', // yellow.3
+        logDate: '#33B3AE', // teal.3
+        logException: '#f8e3a1', // yellow.0
       },
     }),
   },
@@ -90,6 +95,11 @@ const themes: Themes = {
         keyword: '#0077aa',
         type: '#DD4A68',
         variable: '#ee9900',
+        logInfo: '#2da44e', // green.4
+        logError: '#cf222e', // red.5
+        logWarning: '#d4a72c', // yellow.3
+        logDate: '#136061', // teal.6
+        logException: '#7d4e00', // yellow.6
       },
     }),
   },
@@ -104,7 +114,13 @@ const themes: Themes = {
       color: '#586e75',
       backgroundColor: '#44475a',
     },
-    editor: dracula as editor.IStandaloneThemeData,
+    editor: addLogColors(dracula as editor.IStandaloneThemeData, {
+      info: '#50FA7B', // green
+      error: '#FF5555', // red
+      warning: '#FFB86C', // orange
+      date: '#BD93F9', // purple
+      exception: '#F1FA8C', // yellow
+    }),
   },
   'monokai': {
     id: 'monokai',
@@ -117,7 +133,13 @@ const themes: Themes = {
       color: '#49483E',
       backgroundColor: '#3E3D32',
     },
-    editor: monokai as editor.IStandaloneThemeData,
+    editor: addLogColors(monokai as editor.IStandaloneThemeData, {
+      info: '#a6e22e', // green
+      error: '#f92672', // red
+      warning: '#fd971f', // orange
+      date: '#AB9DF2', // purple
+      exception: '#F1FA8C', // yellow
+    }),
   },
   'solarized': {
     id: 'solarized',
@@ -130,7 +152,13 @@ const themes: Themes = {
       color: '#93a1a1', // base1
       backgroundColor: '#073642', // base02
     },
-    editor: solarizedDark as editor.IStandaloneThemeData,
+    editor: addLogColors(solarizedDark as editor.IStandaloneThemeData, {
+      info: '#268bd2', // blue
+      error: '#dc322f', // red
+      warning: '#b58900', // yellow
+      date: '#2aa198', // cyan
+      exception: '#859900', // green
+    }),
   },
   'solarized-light': {
     id: 'solarized-light',
@@ -143,7 +171,13 @@ const themes: Themes = {
       color: '#586e75', // base01
       backgroundColor: '#eee8d5', // base2
     },
-    editor: solarizedLight as editor.IStandaloneThemeData,
+    editor: addLogColors(solarizedLight as editor.IStandaloneThemeData, {
+      info: '#268bd2', // blue
+      error: '#dc322f', // red
+      warning: '#b58900', // yellow
+      date: '#2aa198', // cyan
+      exception: '#859900', // green
+    }),
   },
 };
 
@@ -164,6 +198,11 @@ interface MonacoThemeProps {
     keyword: Color;
     type: Color;
     variable: Color;
+    logInfo: Color;
+    logError: Color;
+    logWarning: Color;
+    logDate: Color;
+    logException: Color;
   };
 }
 
@@ -202,6 +241,11 @@ export function makeMonacoTheme(
       { token: 'identifier', foreground: colors.primary },
       { token: 'type', foreground: colors.type },
       { token: 'comment', foreground: colors.comment },
+      { token: 'info.log', foreground: colors.logInfo },
+      { token: 'error.log', foreground: colors.logError, fontStyle: 'bold' },
+      { token: 'warning.log', foreground: colors.logWarning },
+      { token: 'date.log', foreground: colors.logDate },
+      { token: 'exception.log', foreground: colors.logException },
     ],
     colors: {
       'editor.background': `#${colors.background}`,
@@ -209,3 +253,30 @@ export function makeMonacoTheme(
     },
   };
 }
+
+interface LogColors {
+  info: Color;
+  error: Color;
+  warning: Color;
+  date: Color;
+  exception: Color;
+}
+
+export function addLogColors(
+  theme: editor.IStandaloneThemeData,
+  logColors: LogColors
+): editor.IStandaloneThemeData {
+  const colors = Object.fromEntries(
+    Object.entries(logColors).map(([key, color]) => [key, color.substring(1)])
+  ) as Record<keyof LogColors, string>;
+  theme.rules.push(
+    ...[
+      { token: 'info.log', foreground: colors.info },
+      { token: 'error.log', foreground: colors.error, fontStyle: 'bold' },
+      { token: 'warning.log', foreground: colors.warning },
+      { token: 'date.log', foreground: colors.date },
+      { token: 'exception.log', foreground: colors.exception },
+    ]
+  );
+  return theme;
+}

src/util/constants.ts

@@ -2,3 +2,7 @@ export const bytebinUrl =
   process.env.REACT_APP_BYTEBIN_URL || 'https://bytebin.lucko.me/';
 
 export const postUrl = bytebinUrl + 'post';
+
+export const languageDetectionUrl =
+  process.env.REACT_APP_LANG_DETECT_URL ||
+  'https://language-detection-service.pastes.dev/';

src/util/detect-language.ts

@@ -0,0 +1,49 @@
+import { languageDetectionUrl } from './constants';
+import { Language } from './language';
+
+interface DetectedLanguage {
+  languageId: string;
+  confidence: number;
+}
+
+export async function detectLanguage(id: string): Promise<Language | null> {
+  try {
+    const resp = await fetch(languageDetectionUrl + id);
+    if (resp.ok) {
+      const results = (await resp.json()) as DetectedLanguage[];
+      for (const { languageId, confidence } of results) {
+        if (confidence > 0.5 && lookup[languageId]) {
+          return lookup[languageId];
+        }
+      }
+    }
+  } catch (e) {}
+  return null;
+}
+
+const lookup: Record<string, Language> = {
+  ini: 'log', // the model seems to confidently guess log files as ini - log is the more likely option
+  yaml: 'yaml',
+  md: 'markdown',
+  rb: 'ruby',
+  kt: 'kotlin',
+  xml: 'xml',
+  js: 'javascript',
+  html: 'html',
+  ts: 'typescript',
+  json: 'json',
+  php: 'php',
+  py: 'python',
+  rs: 'rust',
+  sql: 'sql',
+  sh: 'shell',
+  cpp: 'cpp',
+  go: 'go',
+  scala: 'scala',
+  dockerfile: 'dockerfile',
+  java: 'java',
+  cs: 'csharp',
+  css: 'css',
+  groovy: 'java',
+};
+// missing: csv, ml, ex, pas, bat, lua, groovy, v, jl, pm, prolog, matlab, clj, f90, c, tex, coffee, ps1, hs, mm, cmake, erl, dm, dart, asm, makefile, r, swift, lisp, vba, toml, cbl

src/util/highlighting.ts

@@ -1,24 +0,0 @@
-export const languages = {
-  config: ['yaml', 'json', 'xml', 'ini'],
-  code: [
-    'java',
-    'javascript',
-    'typescript',
-    'python',
-    'kotlin',
-    'cpp',
-    'csharp',
-    'shell',
-    'ruby',
-    'rust',
-    'sql',
-    'go',
-  ],
-  web: ['html', 'css', 'php'],
-  misc: ['plain', 'dockerfile', 'markdown'],
-};
-
-// missing following the rewrite: toml, properties, log, javastacktrace, groovy, haskell, protobuf
-// would be good to add these back with custom language definitions
-
-export const languageIds = Object.values(languages).flat(1);

src/util/language.ts

@@ -0,0 +1,70 @@
+export type Language =
+  | 'plain'
+  | 'plaintext'
+  | 'log'
+  | 'yaml'
+  | 'json'
+  | 'xml'
+  | 'ini'
+  | 'java'
+  | 'javascript'
+  | 'typescript'
+  | 'python'
+  | 'kotlin'
+  | 'scala'
+  | 'cpp'
+  | 'csharp'
+  | 'shell'
+  | 'ruby'
+  | 'rust'
+  | 'sql'
+  | 'go'
+  | 'html'
+  | 'css'
+  | 'scss'
+  | 'php'
+  | 'graphql'
+  | 'dockerfile'
+  | 'markdown'
+  | 'proto';
+
+export const unknownLanguage: Language & 'plain' = 'plain';
+
+export interface Languages {
+  text: Language[];
+  config: Language[];
+  code: Language[];
+  web: Language[];
+  misc: Language[];
+}
+
+export const languages: Languages = {
+  text: ['plaintext', 'log'],
+  config: ['yaml', 'json', 'xml', 'ini'],
+  code: [
+    'java',
+    'javascript',
+    'typescript',
+    'python',
+    'kotlin',
+    'scala',
+    'cpp',
+    'csharp',
+    'shell',
+    'ruby',
+    'rust',
+    'sql',
+    'go',
+  ],
+  web: ['html', 'css', 'scss', 'php', 'graphql'],
+  misc: ['dockerfile', 'markdown', 'proto'],
+};
+
+export const languageIds: Language[] = [
+  ...Object.values(languages).flat(1),
+  unknownLanguage,
+];
+
+export function isLanguage(lang: string): lang is Language {
+  return languageIds.includes(lang as Language);
+}

src/util/log-language.ts

@@ -0,0 +1,71 @@
+import type { languages } from 'monaco-editor';
+
+// Source:
+// - https://github.com/emilast/vscode-logfile-highlighter/blob/master/syntaxes/log.tmLanguage
+// - https://github.com/sumy7/monaco-language-log/blob/main/language-log.js
+export const logLanguage: languages.IMonarchLanguage = {
+  defaultToken: '',
+  tokenizer: {
+    // prettier-ignore
+    root: [
+      // Trace/Verbose
+      [/\b(Trace)\b:/, 'verbose'],
+      // Serilog VERBOSE
+      [/\[(verbose|verb|vrb|vb|v)]/i, 'verbose'],
+      // Android logcat Verbose
+      [/\bV\//, 'verbose'],
+      // DEBUG
+      [/\b(DEBUG|Debug)\b|\b([dD][eE][bB][uU][gG]):/, 'debug'],
+      // Serilog DEBUG
+      [/\[(debug|dbug|dbg|de|d)]/i, 'debug'],
+      // Android logcat Debug
+      [/\bD\//, 'debug'],
+      // INFO
+      [/\b(HINT|INFO|INFORMATION|Info|NOTICE|II)\b|\b([iI][nN][fF][oO]|[iI][nN][fF][oO][rR][mM][aA][tT][iI][oO][nN]):/, 'info'],
+      // serilog INFO
+      [/\[(information|info|inf|in|i)]/i, 'info'],
+      // Android logcat Info
+      [/\bI\//, 'info'],
+      // WARN
+      [/\b(WARNING|WARN|Warn|WW)\b|\b([wW][aA][rR][nN][iI][nN][gG]):/, 'warning'],
+      // Serilog WARN
+      [/\[(warning|warn|wrn|wn|w)]/i, 'warning'],
+      // Android logcat Warning
+      [/\bW\//, 'warning'],
+      // ERROR
+      [/\b(ALERT|CRITICAL|EMERGENCY|ERROR|FAILURE|FAIL|Fatal|FATAL|Error|EE)\b|\b([eE][rR][rR][oO][rR]):/, 'error'],
+      // Serilog ERROR
+      [/\[(error|eror|err|er|e|fatal|fatl|ftl|fa|f)]/i, 'error'],
+      // Android logcat Error
+      [/\bE\//, 'error'],
+      // ISO dates ("2020-01-01")
+      [/\b\d{4}-\d{2}-\d{2}(T|\b)/, 'date'],
+      // Culture specific dates ("01/01/2020", "01.01.2020")
+      [/\b\d{2}[^\w\s]\d{2}[^\w\s]\d{4}\b/, 'date'],
+      // Clock times with optional timezone ("01:01:01", "01:01:01.001", "01:01:01+01:01")
+      [/\d{1,2}:\d{2}(:\d{2}([.,]\d{1,})?)?(Z| ?[+-]\d{1,2}:\d{2})?\b/, 'date'],
+      // Git commit hashes of length 40, 10, or 7
+      //[/\b([0-9a-fA-F]{40}|[0-9a-fA-F]{10}|[0-9a-fA-F]{7})\b/, 'constant'],
+      // Guids
+      [/[0-9a-fA-F]{8}[-]?([0-9a-fA-F]{4}[-]?){3}[0-9a-fA-F]{12}/, 'constant'],
+      // MAC addresses: 89:A1:23:45:AB:C0, fde8:e767:269c:0:9425:3477:7c8f:7f1a
+      //[/\b([0-9a-fA-F]{2,}[:-])+([0-9a-fA-F]{2,})+\b/, 'constant'],
+      // Constants
+      //[/\b([0-9]+|true|false|null)\b/, 'constant'],
+      // Hex Constants
+      [/\b(0x[a-fA-F0-9]+)\b/, 'constant'],
+      // String constants
+      [/"[^"]*"/, 'string'],
+      [/(?<![\w])'[^']*'/, 'string'],
+      // Colorize rows of exception call stacks
+      [/[\t ]*at[\t ]+.*$/, 'exception'],
+      [/Exception in thread ".*" .*$/, 'exception'],
+      // Exception type names
+      [/\b([a-zA-Z.]*Exception)\b/, 'exception'],
+      // Match Urls
+      [/\b(http|https|ftp|file):\/\/\S+\b\/?/, 'constant'],
+      // Match character and . sequences (such as namespaces) as well as file names and extensions (e.g. bar.txt)
+      //[/(?<![\w/\\])([\w-]+\.)+([\w-])+(?![\w/\\])/, 'constant'],
+    ],
+  },
+};

src/util/storage.ts

@@ -1,12 +1,12 @@
 import { gzip } from 'pako';
 import MIMEType from 'whatwg-mimetype';
 import { bytebinUrl, postUrl } from './constants';
-import { languageIds } from './highlighting';
+import { isLanguage, Language } from './language';
 
 interface LoadResultSuccess {
   ok: true;
   content: string;
-  type?: string;
+  type?: Language;
 }
 
 interface LoadResultFail {
@@ -64,13 +64,21 @@ export async function saveToBytebin(
   return null;
 }
 
-export function contentTypeToLanguage(contentType: string) {
+export function contentTypeToLanguage(
+  contentType: string
+): Language | undefined {
   const { type, subtype: subType } = new MIMEType(contentType);
   if (type === 'application' && subType === 'json') {
     return 'json';
   }
-  if (type === 'text' && languageIds.includes(subType.toLowerCase())) {
+
-    return subType.toLowerCase();
+  let subTypeLower = subType.toLowerCase();
+  if (subTypeLower.startsWith('x-')) {
+    subTypeLower = subTypeLower.substring(2);
+  }
+
+  if (type === 'text' && isLanguage(subTypeLower)) {
+    return subTypeLower;
   }
 }