Prepare release 1.4.0 (#1344)

Prepare release 1.4.0

.dockerignore

+.idea
+coverage
+node_modules/
+
+# ignore config files
+config.json
+.sequelizerc
+
+# ignore webpack build
+public/build
+public/views/build
+
+.nyc_output
+coverage/

.eslintignore

-lib/ot
-public/vendor
-public/build

.eslintrc.js

-module.exports = {
-  "root": true,
-  "extends": "standard",
-  "env": {
-    "node": true
-  },
-  "rules": {
-    // at some point all of these should return to their default "error" state
-    // but right now, this is not a good choice, because too many places are
-    // wrong.
-    "import/first": ["warn"],
-    "indent": ["warn"],
-    "no-multiple-empty-lines": ["warn"],
-    "no-multi-spaces": ["warn"],
-    "object-curly-spacing": ["warn"],
-    "one-var": ["warn"],
-    "quotes": ["warn"],
-    "semi": ["warn"],
-    "space-infix-ops": ["warn"]
-  }
-};

.gitignore

@@ -27,3 +27,5 @@ public/views/build
 
 public/uploads/*
 !public/uploads/.gitkeep
+/.nyc_output
+/coverage/

.sequelizerc.example

-var path = require('path');
+const path = require('path')
+const config = require('./lib/config')
 
 module.exports = {
-    'config':          path.resolve('config.json'),
-    'migrations-path': path.resolve('lib', 'migrations'),
-    'models-path':     path.resolve('lib', 'models'),
-    'url':             'change this'
-}
\ No newline at end of file
+  config: path.resolve('config.json'),
+  'migrations-path': path.resolve('lib', 'migrations'),
+  'models-path': path.resolve('lib', 'models'),
+  url: process.env['CMD_DB_URL'] || config.dbURL
+}

.travis.yml

 language: node_js
-dist: trusty
+
+node_js:
+  - "lts/carbon"
+  - "lts/dubnium"
+  - "11"
+  - "12"
+
+dist: xenial
 cache: yarn
-env:
-  global:
-    - CXX=g++-4.8
-    - YARN_VERSION=1.3.2
+
+matrix:
+  fast_finish: true
+  include:
+    - node_js: lts/carbon
+    - node_js: lts/dubnium
+  allow_failures:
+    - node_js: "11"
+    - node_js: "12"
+
+script:
+  - yarn test:ci
+  - yarn build
 
 jobs:
   include:
-    - env: task=npm-test
-      node_js:
-        - 8
-      before_install:
-        - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version "$YARN_VERSION"
-        - export PATH="$HOME/.yarn/bin:$PATH"
-    - env: task=npm-test
-      node_js:
-        - 10
-      before_install:
-        - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version "$YARN_VERSION"
-        - export PATH="$HOME/.yarn/bin:$PATH"
-    - env: task=ShellCheck
-      script:
-        - shellcheck bin/heroku bin/setup
-      language: generic
-    - env: task=doctoc
-      install: npm install doctoc
+    - stage: doctoc-check
+      install: npm install -g doctoc
+      if: type = pull_request OR branch = master
       script:
         - cp README.md README.md.orig
         - npm run doctoc
         - diff -q README.md README.md.orig
-      language: generic
-    - env: task=json-lint
-      addons:
-        apt:
-          packages:
-          - jq
-      script:
-        - npm run jsonlint
-      language: generic
+      node_js: lts/carbon

README.md

@@ -9,6 +9,8 @@ CodiMD
 CodiMD lets you collaborate in real-time with markdown.
 Built on [HackMD](https://hackmd.io) source code, CodiMD lets you host and control your team's content with speed and ease.
 
+![screenshot](https://raw.githubusercontent.com/hackmdio/codimd/develop/public/screenshot.png)
+
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 # Table of Contents
@@ -71,20 +73,11 @@ All contributions are welcome! Even asking a question helps.
 ## Browser Support
 
 CodiMD is a service that runs on Node.js, while users use the service through browsers. We support your users using the following browsers: 
-- ![Chrome](http://browserbadge.com/chrome/47/18px)
-    - Chrome >= 47
-    - Chrome for Android >= 47
-- ![Safari](http://browserbadge.com/safari/9/18px)
-    - Safari >= 9
-    - iOS Safari >= 8.4
-- ![Firefox](http://browserbadge.com/firefox/44/18px)
-    - Firefox >= 44
-- ![IE](http://browserbadge.com/ie/9/18px)
-    - IE >= 9
-    - Edge >= 12
-- ![Opera](http://browserbadge.com/opera/34/18px)
-    - Opera >= 34
-    - Opera Mini not supported
+- <img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/chrome/chrome_48x48.png" alt="Chrome" width="24px" height="24px" /> Chrome >= 47, Chrome for Android >= 47
+- <img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/safari/safari_48x48.png" alt="Safari" width="24px" height="24px" /> Safari >= 9, iOS Safari >= 8.4
+- <img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/firefox/firefox_48x48.png" alt="Firefox" width="24px" height="24px" /> Firefox >= 44
+- <img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/edge/edge_48x48.png" alt="IE / Edge" width="24px" height="24px" /> IE >= 9, Edge >= 12
+- <img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/opera/opera_48x48.png" alt="Opera" width="24px" height="24px" /> Opera >= 34, Opera Mini not supported
 - Android Browser >= 4.4
 
 To stay up to date with your installation it's recommended to subscribe the [release feed][github-release-feed].

app.js

@@ -7,7 +7,6 @@ var ejs = require('ejs')
 var passport = require('passport')
 var methodOverride = require('method-override')
 var cookieParser = require('cookie-parser')
-var compression = require('compression')
 var session = require('express-session')
 var SequelizeStore = require('connect-session-sequelize')(session.Store)
 var fs = require('fs')
@@ -26,34 +25,37 @@ var response = require('./lib/response')
 var models = require('./lib/models')
 var csp = require('./lib/csp')
 
-// server setup
-var app = express()
-var server = null
-if (config.useSSL) {
-  var ca = (function () {
-    var i, len, results
-    results = []
-    for (i = 0, len = config.sslCAPath.length; i < len; i++) {
-      results.push(fs.readFileSync(config.sslCAPath[i], 'utf8'))
+function createHttpServer () {
+  if (config.useSSL) {
+    const ca = (function () {
+      let i, len
+      const results = []
+      for (i = 0, len = config.sslCAPath.length; i < len; i++) {
+        results.push(fs.readFileSync(config.sslCAPath[i], 'utf8'))
+      }
+      return results
+    })()
+    const options = {
+      key: fs.readFileSync(config.sslKeyPath, 'utf8'),
+      cert: fs.readFileSync(config.sslCertPath, 'utf8'),
+      ca: ca,
+      dhparam: fs.readFileSync(config.dhParamPath, 'utf8'),
+      requestCert: false,
+      rejectUnauthorized: false
     }
-    return results
-  })()
-  var options = {
-    key: fs.readFileSync(config.sslKeyPath, 'utf8'),
-    cert: fs.readFileSync(config.sslCertPath, 'utf8'),
-    ca: ca,
-    dhparam: fs.readFileSync(config.dhParamPath, 'utf8'),
-    requestCert: false,
-    rejectUnauthorized: false
+    return require('https').createServer(options, app)
+  } else {
+    return require('http').createServer(app)
   }
-  server = require('https').createServer(options, app)
-} else {
-  server = require('http').createServer(app)
 }
 
+// server setup
+var app = express()
+var server = createHttpServer()
+
 // logger
 app.use(morgan('combined', {
-  'stream': logger.stream
+  stream: logger.stream
 }))
 
 // socket io
@@ -77,9 +79,6 @@ var sessionStore = new SequelizeStore({
   db: models.sequelize
 })
 
-// compression
-app.use(compression())
-
 // use hsts to tell https users stick to this
 if (config.hsts.enable) {
   app.use(helmet.hsts({
@@ -181,6 +180,7 @@ app.locals.serverURL = config.serverURL
 app.locals.sourceURL = config.sourceURL
 app.locals.allowAnonymous = config.allowAnonymous
 app.locals.allowAnonymousEdits = config.allowAnonymousEdits
+app.locals.permission = config.permission
 app.locals.allowPDFExport = config.allowPDFExport
 app.locals.authProviders = {
   facebook: config.isFacebookEnable,
@@ -279,6 +279,7 @@ process.on('uncaughtException', function (err) {
 function handleTermSignals () {
   logger.info('CodiMD has been killed by signal, try to exit gracefully...')
   realtime.maintenance = true
+  realtime.terminate()
   // disconnect all socket.io clients
   Object.keys(io.sockets.sockets).forEach(function (key) {
     var socket = io.sockets.sockets[key]
@@ -299,6 +300,9 @@ function handleTermSignals () {
       })
     }
   }, 100)
+  setTimeout(() => {
+    process.exit(1)
+  }, 5000)
 }
 process.on('SIGINT', handleTermSignals)
 process.on('SIGTERM', handleTermSignals)

app.json

@@ -11,18 +11,10 @@
     "logo": "https://github.com/hackmdio/codimd/raw/master/public/codimd-icon-1024.png",
     "success_url": "/",
     "env": {
-        "BUILD_ASSETS": {
-            "description": "Our build script variable",
-            "value": "true"
-        },
         "NPM_CONFIG_PRODUCTION": {
             "description": "Let npm also install development build tool",
             "value": "false"
         },
-        "DB_TYPE": {
-            "description": "Specify database type. See sequelize available databases. Default using postgres",
-            "value": "postgres"
-        },
         "HMD_SESSION_SECRET": {
             "description": "Secret used to secure session cookies.",
             "required": false

bin/heroku

@@ -2,7 +2,7 @@
 
 set -e
 
-if [ "$BUILD_ASSETS" = true ]; then
+if [ ! -z "$DYNO" ]; then
   # setup config files
   cat << EOF > .sequelizerc
 var path = require('path');
@@ -11,8 +11,7 @@ module.exports = {
     'config':          path.resolve('config.json'),
     'migrations-path': path.resolve('lib', 'migrations'),
     'models-path':     path.resolve('lib', 'models'),
-    'url':             process.env.DATABASE_URL,
-    'dialect':         process.env.DB_TYPE
+    'url':             process.env.DATABASE_URL
 }
 
 EOF
@@ -26,6 +25,4 @@ EOF
 
 EOF
 
-  # build app
-  npm run build
 fi

config.json.example

@@ -3,7 +3,8 @@
         "db": {
             "dialect": "sqlite",
             "storage": ":memory:"
-        }
+        },
+        "linkifyHeaderStyle": "gfm"
     },
     "development": {
         "loglevel": "debug",
@@ -13,7 +14,8 @@
         "db": {
             "dialect": "sqlite",
             "storage": "./db.codimd.sqlite"
-        }
+        },
+        "linkifyHeaderStyle": "gfm"
     },
     "production": {
         "domain": "localhost",
@@ -123,6 +125,11 @@
         {
           "connectionString": "change this",
           "container": "change this"
-        }
+        },
+        "plantuml":
+        {
+          "server": "https://www.plantuml.com/plantuml"
+        },
+        "linkifyHeaderStyle": "gfm"
     }
 }

deployments/Dockerfile

+FROM hackmdio/buildpack:1.0.4 as BUILD
+
+COPY --chown=hackmd:hackmd . .
+
+RUN set -xe && \
+    git reset --hard && \
+    git clean -fx && \
+    yarn install && \
+    yarn build && \
+    yarn install --production=true && \
+    cp ./deployments/docker-entrypoint.sh ./ && \
+    cp .sequelizerc.example .sequelizerc && \
+    rm -rf .git .gitignore .travis.yml .dockerignore .editorconfig .babelrc .mailmap .sequelizerc.example \
+        test docs contribute \
+        yarn.lock webpack.prod.js webpack.htmlexport.js webpack.dev.js webpack.common.js \
+        config.json.example README.md CONTRIBUTING.md AUTHORS
+
+FROM hackmdio/runtime:1.0.4
+USER hackmd
+WORKDIR /home/hackmd/app
+COPY --chown=1500:1500 --from=BUILD /home/hackmd/app .
+EXPOSE 3000
+ENTRYPOINT ["/home/hackmd/app/docker-entrypoint.sh"]

deployments/build.sh

+#!/usr/bin/env bash
+
+CURRENT_DIR=$(dirname "$BASH_SOURCE")
+
+docker build -t hackmdio/codimd -f "$CURRENT_DIR/Dockerfile" "$CURRENT_DIR/.."

deployments/docker-compose.yml

+version: "3"
+services:
+  database:
+    image: postgres:11.5
+    environment:
+      - POSTGRES_USER=codimd
+      - POSTGRES_PASSWORD=change_password
+      - POSTGRES_DB=codimd
+    volumes:
+      - "database-data:/var/lib/postgresql/data"
+    restart: always
+  codimd:
+    build:
+      context: ..
+      dockerfile: ./deployments/Dockerfile
+    environment:
+      - CMD_DB_URL=postgres://codimd:change_password@database/codimd
+      - CMD_USECDN=false
+    depends_on:
+      - database
+    ports:
+      - "3000:3000"
+    volumes:
+      - upload-data:/home/hackmd/app/public/uploads
+    restart: always
+volumes:
+  database-data: {}
+  upload-data: {}

deployments/docker-entrypoint.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ "$#" -gt 0 ]]; then
+    exec "$@"
+    exit $?
+fi
+
+# check database and redis is ready
+pcheck -constr "$CMD_DB_URL"
+
+# run DB migrate
+NEED_MIGRATE=${CMD_AUTO_MIGRATE:=true}
+
+if [[ "$NEED_MIGRATE" = "true" ]] && [[ -f .sequelizerc ]] ; then
+    npx sequelize db:migrate
+fi
+
+# start application
+node app.js

lib/config/default.js

@@ -31,6 +31,7 @@ module.exports = {
   useCDN: true,
   allowAnonymous: true,
   allowAnonymousEdits: false,
+  allowAnonymousViews: true,
   allowFreeURL: false,
   forbiddenNoteIDs: ['robots.txt', 'favicon.ico', 'api'],
   defaultPermission: 'editable',
@@ -60,8 +61,11 @@ module.exports = {
   responseMaxLag: 70,
   // document
   documentMaxLength: 100000,
-  // image upload setting, available options are imgur/s3/filesystem/azure
+  // image upload setting, available options are imgur/s3/filesystem/azure/lutim
   imageUploadType: 'filesystem',
+  lutim: {
+    url: 'https://framapic.org/'
+  },
   imgur: {
     clientID: undefined
   },
@@ -100,6 +104,7 @@ module.exports = {
     consumerSecret: undefined
   },
   github: {
+    enterpriseURL: undefined, // if you use github.com, not need to specify
     clientID: undefined,
     clientSecret: undefined
   },
@@ -151,9 +156,27 @@ module.exports = {
       email: undefined
     }
   },
+  plantuml: {
+    server: 'https://www.plantuml.com/plantuml'
+  },
   email: true,
   allowEmailRegister: true,
   allowGravatar: true,
   allowPDFExport: true,
-  openID: false
+  openID: false,
+  defaultUseHardbreak: true,
+  // linkifyHeaderStyle - How is a header text converted into a link id.
+  // Header Example: "3.1. Good Morning my Friend! - Do you have 5$?"
+  // * 'keep-case' is the legacy CodiMD value.
+  //    Generated id: "31-Good-Morning-my-Friend---Do-you-have-5"
+  // * 'lower-case' is the same like legacy (see above), but converted to lower-case.
+  //    Generated id: "#31-good-morning-my-friend---do-you-have-5"
+  // * 'gfm' _GitHub-Flavored Markdown_ style as described here:
+  //    https://gist.github.com/asabaylus/3071099#gistcomment-1593627
+  //    It works like 'lower-case', but making sure the ID is unique.
+  //    This is What GitHub, GitLab and (hopefully) most other tools use.
+  //    Generated id:   "31-good-morning-my-friend---do-you-have-5"
+  //    2nd appearance: "31-good-morning-my-friend---do-you-have-5-1"
+  //    3rd appearance: "31-good-morning-my-friend---do-you-have-5-2"
+  linkifyHeaderStyle: 'keep-case'
 }

lib/config/environment.js

@@ -27,6 +27,7 @@ module.exports = {
   useCDN: toBooleanConfig(process.env.CMD_USECDN),
   allowAnonymous: toBooleanConfig(process.env.CMD_ALLOW_ANONYMOUS),
   allowAnonymousEdits: toBooleanConfig(process.env.CMD_ALLOW_ANONYMOUS_EDITS),
+  allowAnonymousViews: toBooleanConfig(process.env.CMD_ALLOW_ANONYMOUS_VIEWS),
   allowFreeURL: toBooleanConfig(process.env.CMD_ALLOW_FREEURL),
   forbiddenNoteIDs: toArrayConfig(process.env.CMD_FORBIDDEN_NOTE_IDS),
   defaultPermission: process.env.CMD_DEFAULT_PERMISSION,
@@ -65,6 +66,7 @@ module.exports = {
     consumerSecret: process.env.CMD_TWITTER_CONSUMERSECRET
   },
   github: {
+    enterpriseURL: process.env.CMD_GITHUB_ENTERPRISE_URL,
     clientID: process.env.CMD_GITHUB_CLIENTID,
     clientSecret: process.env.CMD_GITHUB_CLIENTSECRET
   },
@@ -127,9 +129,14 @@ module.exports = {
       email: process.env.CMD_SAML_ATTRIBUTE_EMAIL
     }
   },
+  plantuml: {
+    server: process.env.CMD_PLANTUML_SERVER
+  },
   email: toBooleanConfig(process.env.CMD_EMAIL),
   allowEmailRegister: toBooleanConfig(process.env.CMD_ALLOW_EMAIL_REGISTER),
   allowGravatar: toBooleanConfig(process.env.CMD_ALLOW_GRAVATAR),
   allowPDFExport: toBooleanConfig(process.env.CMD_ALLOW_PDF_EXPORT),
-  openID: toBooleanConfig(process.env.CMD_OPENID)
+  openID: toBooleanConfig(process.env.CMD_OPENID),
+  defaultUseHardbreak: toBooleanConfig(process.env.CMD_DEFAULT_USE_HARD_BREAK),
+  linkifyHeaderStyle: process.env.CMD_LINKIFY_HEADER_STYLE
 }

lib/config/index.js

@@ -53,14 +53,14 @@ if (['debug', 'verbose', 'info', 'warn', 'error'].includes(config.loglevel)) {
 
 // load LDAP CA
 if (config.ldap.tlsca) {
-  let ca = config.ldap.tlsca.split(',')
-  let caContent = []
-  for (let i of ca) {
-    if (fs.existsSync(i)) {
-      caContent.push(fs.readFileSync(i, 'utf8'))
+  const certificateAuthorities = config.ldap.tlsca.split(',')
+  const caContent = []
+  for (const ca of certificateAuthorities) {
+    if (fs.existsSync(ca)) {
+      caContent.push(fs.readFileSync(ca, 'utf8'))
     }
   }
-  let tlsOptions = {
+  const tlsOptions = {
     ca: caContent
   }
   config.ldap.tlsOptions = config.ldap.tlsOptions ? Object.assign(config.ldap.tlsOptions, tlsOptions) : tlsOptions
@@ -68,11 +68,17 @@ if (config.ldap.tlsca) {
 
 // Permission
 config.permission = Permission
-if (!config.allowAnonymous && !config.allowAnonymousEdits) {
+let defaultPermission = config.permission.editable
+if (!config.allowAnonymous && !config.allowAnonymousViews) {
+  delete config.permission.freely
+  delete config.permission.editable
+  delete config.permission.locked
+  defaultPermission = config.permission.limited
+} else if (!config.allowAnonymous && !config.allowAnonymousEdits) {
   delete config.permission.freely
 }
 if (!(config.defaultPermission in config.permission)) {
-  config.defaultPermission = config.permission.editable
+  config.defaultPermission = defaultPermission
 }
 
 // cache result, cannot change config in runtime!!!
@@ -134,10 +140,10 @@ config.isGitlabSnippetsEnable = (!config.gitlab.scope || config.gitlab.scope ===
 config.updateI18nFiles = (env === Environment.development)
 
 // merge legacy values
-let keys = Object.keys(config)
+const keys = Object.keys(config)
 const uppercase = /[A-Z]/
 for (let i = keys.length; i--;) {
-  let lowercaseKey = keys[i].toLowerCase()
+  const lowercaseKey = keys[i].toLowerCase()
   // if the config contains uppercase letters
   // and a lowercase version of this setting exists
   // and the config with uppercase is not set
@@ -164,8 +170,8 @@ if (config.sessionSecret === 'secret') {
 }
 
 // Validate upload upload providers
-if (['filesystem', 's3', 'minio', 'imgur', 'azure'].indexOf(config.imageUploadType) === -1) {
-  logger.error('"imageuploadtype" is not correctly set. Please use "filesystem", "s3", "minio", "azure" or "imgur". Defaulting to "filesystem"')
+if (['filesystem', 's3', 'minio', 'imgur', 'azure', 'lutim'].indexOf(config.imageUploadType) === -1) {
+  logger.error('"imageuploadtype" is not correctly set. Please use "filesystem", "s3", "minio", "azure", "lutim" or "imgur". Defaulting to "filesystem"')
   config.imageUploadType = 'filesystem'
 }
 
@@ -194,6 +200,7 @@ config.sslCAPath.forEach(function (capath, i, array) {
   array[i] = path.resolve(appRootPath, capath)
 })
 
+config.appRootPath = appRootPath
 config.sslCertPath = path.resolve(appRootPath, config.sslCertPath)
 config.sslKeyPath = path.resolve(appRootPath, config.sslKeyPath)
 config.dhParamPath = path.resolve(appRootPath, config.dhParamPath)

lib/history.js

@@ -9,14 +9,6 @@ var logger = require('./logger')
 var response = require('./response')
 var models = require('./models')
 
-// public
-var History = {
-  historyGet: historyGet,
-  historyPost: historyPost,
-  historyDelete: historyDelete,
-  updateHistory: updateHistory
-}
-
 function getHistory (userid, callback) {
   models.User.findOne({
     where: {
@@ -41,7 +33,7 @@ function getHistory (userid, callback) {
           continue
         }
         try {
-          let id = LZString.decompressFromBase64(history[i].id)
+          const id = LZString.decompressFromBase64(history[i].id)
           if (id && models.Note.checkNoteIdValid(id)) {
             history[i].id = models.Note.encodeNoteId(id)
           }
@@ -200,4 +192,8 @@ function historyDelete (req, res) {
   }
 }
 
-module.exports = History
+// public
+exports.historyGet = historyGet
+exports.historyPost = historyPost
+exports.historyDelete = historyDelete
+exports.updateHistory = updateHistory