GC, Seed and User management (#52)

* Update sql to map uuid to token

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Fix millis to secs conversion

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Add synchronisation to media enpoint DB access

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Change error code for rate limiter

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* move prepared statements to Thread Local Storage

* Change test end points

* init GC

* Add GC

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Change status return

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Auto generate token in db

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Remove user management and rate limiting

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Add seed for random number generator

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Store random instance as class member

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Update locustfile

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Add API documentation

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Move updateTimeStamp to getChallenge methdod
Remove user tables for the DB

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Update Timestamp when creating mapId entry

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Add request method type

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

* Minor fixes

Signed-off-by: Rahul Rudragoudar <rr83019@gmail.com>

README.md

@@ -44,6 +44,55 @@ An image of a word is blurred before being shown to the user.
 ### LabelCaptcha
 An image that has a pair of words is created. The answer to one of the words is known and to that of the other is unknown. The user is tested on the known word, and their answer to the unknown word is recorded. If a sufficient number of users agree on their answer to the unknown word, it is transferred to the list of known words.
 
+***
+
+## HTTP API 
+### - `/captcha`: `POST`
+  - Parameters:
+    - `level`: `String` - 
+      The difficulty level of a captcha
+       - easy
+       - medium
+       - hard
+    - `input_type`: `String` - 
+      The type of input option for a captcha
+       - text
+       - click (choose)
+    - `media`: `String` - 
+      The type of media of a captcha
+       - image
+       - audio
+       - gif
+    - `size`: `dict` - 
+      The dimensions of a captcha (Optional). It needs two more fields nested in this parameter
+       - `height`: `Int`
+       - `width`: `Int`
+
+  - Return type:
+    - `id`: `String` - The uuid of the captcha generated
+
+
+### - `/media`: `POST`,`GET` 
+  - Parameters:
+    - `id`: `String` - The uuid of the captcha
+
+  - Return type:
+    - `image`: `Array[Byte]` - The requested media as bytes
+
+
+### - `/answer`: `POST`
+  - Parameter:
+    - `id`: `String` - The uuid of the captcha that needs to be solved
+    - `answer`: `String` - The answer to the captcha that needs to be validated
+
+  - Return Type:
+    - `result`: `String` - The result after validation/checking of the answer
+      - True - If the answer is correct
+      - False - If the answer is incorrect
+      - Expired - If the time limit to solve the captcha exceeds
+
+***
+
 ## Roadmap
 
 Things to do in the future:

src/main/scala/lc/Main.scala

@@ -6,6 +6,8 @@ import java.util.concurrent._
 import java.util.UUID
 import java.sql.{Blob, ResultSet}
 import java.util.concurrent.atomic.AtomicInteger
+import java.io._
+import java.sql.Statement
 
 case class Size(height: Int, width: Int)
 case class Parameters(level: String, media: String, input_type: String, size: Option[Size])
@@ -17,11 +19,11 @@ case class ProviderSecret(provider: String, secret: String)
 object CaptchaProviders {
   val providers = Map(
     "FilterChallenge" -> new FilterChallenge,
-    "FontFunCaptcha" -> new FontFunCaptcha,
+    // "FontFunCaptcha" -> new FontFunCaptcha,
     "GifCaptcha" -> new GifCaptcha,
     "ShadowTextCaptcha" -> new ShadowTextCaptcha,
     "RainDropsCaptcha" -> new RainDropsCP,
-    "LabelCaptcha" -> new LabelCaptcha
+    // "LabelCaptcha" -> new LabelCaptcha
     )
 
   def generateChallengeSamples() = {
@@ -31,80 +33,98 @@ object CaptchaProviders {
   }
 }
 
+class Statements(dbConn: DBConn) {
+  val insertPstmt = dbConn.con.prepareStatement("INSERT INTO challenge(id, secret, provider, contentType, image) VALUES (?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS )
+  val mapPstmt = dbConn.con.prepareStatement("INSERT INTO mapId(uuid, token, lastServed) VALUES (?, ?, CURRENT_TIMESTAMP)")
+  val selectPstmt = dbConn.con.prepareStatement("SELECT secret, provider FROM challenge WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ? AND DATEDIFF(MINUTE, DATEADD(MINUTE,2,m.lastServed), CURRENT_TIMESTAMP) <= 0)")
+  val imagePstmt = dbConn.con.prepareStatement("SELECT image FROM challenge c, mapId m WHERE c.token=m.token AND m.uuid = ?")
+  val updateSolvedPstmt = dbConn.con.prepareStatement("UPDATE challenge SET solved = solved+1 WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ?)")
+  val tokenPstmt = dbConn.con.prepareStatement("SELECT token FROM challenge WHERE solved < 10 ORDER BY RAND() LIMIT 1")
+}
+
+object Statements {
+  var dbConn: DBConn = _
+  val tlStmts = ThreadLocal.withInitial(() => new Statements(dbConn))
+}
+
 class Captcha(throttle: Int, dbConn: DBConn) {
   import CaptchaProviders._
 
   private val stmt = dbConn.getStatement()
-  stmt.execute("CREATE TABLE IF NOT EXISTS challenge(token varchar, id varchar, secret varchar, provider varchar, contentType varchar, image blob, solved boolean default False, PRIMARY KEY(token))")
+  stmt.execute("CREATE TABLE IF NOT EXISTS challenge(token int auto_increment, id varchar, secret varchar, provider varchar, contentType varchar, image blob, solved int default 0, PRIMARY KEY(token))")
-  stmt.execute("CREATE TABLE IF NOT EXISTS mapId(uuid varchar, token varchar, PRIMARY KEY(uuid), FOREIGN KEY(token) REFERENCES challenge(token))")
+  stmt.execute("CREATE TABLE IF NOT EXISTS mapId(uuid varchar, token int, lastServed timestamp, PRIMARY KEY(uuid), FOREIGN KEY(token) REFERENCES challenge(token) ON DELETE CASCADE)")
-  stmt.execute("CREATE TABLE IF NOT EXISTS users(email varchar, hash int)")
 
-  private val insertPstmt = dbConn.con.prepareStatement("INSERT INTO challenge(token, id, secret, provider, contentType, image) VALUES (?, ?, ?, ?, ?, ?)")
+  private val seed = System.currentTimeMillis.toString.substring(2,6).toInt
-  private val mapPstmt = dbConn.con.prepareStatement("INSERT INTO mapId(uuid, token) VALUES (?, ?)")
+  private val random = new scala.util.Random(seed)
-  private val selectPstmt = dbConn.con.prepareStatement("SELECT secret, provider FROM challenge WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ?)")
-  private val imagePstmt = dbConn.con.prepareStatement("SELECT image FROM challenge c, mapId m WHERE c.token=m.token AND m.uuid = ?")
-  private val updatePstmt = dbConn.con.prepareStatement("UPDATE challenge SET solved = True WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ?)")
-  private val userPstmt = dbConn.con.prepareStatement("INSERT INTO users(email, hash) VALUES (?,?)")
 
+  def getNextRandomInt(max: Int) = random.synchronized {
+    random.nextInt(max)
+  }
+  
   def getProvider(): String = {
-    val random = new scala.util.Random
     val keys = providers.keys
-    val providerIndex = keys.toVector(random.nextInt(keys.size))
+    val providerIndex = keys.toVector(getNextRandomInt(keys.size))
     providerIndex
   }
 
   def getCaptcha(id: Id): Array[Byte] = {
     var image :Array[Byte] = null
     var blob: Blob = null
-    val imageOpt = imagePstmt.synchronized {
+    try {
+      val imagePstmt = Statements.tlStmts.get.imagePstmt
     	imagePstmt.setString(1, id.id)
     	val rs: ResultSet = imagePstmt.executeQuery()
     	if(rs.next()){
-        blob = rs.getBlob("image")
+          blob = rs.getBlob("image")
-        updatePstmt.synchronized{
-          updatePstmt.setString(1,id.id)
-          updatePstmt.executeUpdate()
-        }
-      }
     	if(blob != null)
     		image =  blob.getBytes(1, blob.length().toInt)
     	image
     }
-    imageOpt
+    image
+  } catch{ case e: Exception =>
+    println(e)
+    image
+  }
   }
 
   private val uniqueIntCount = new AtomicInteger()
 
-  def generateChallenge(param: Parameters): String = {
+  def generateChallenge(param: Parameters): Int = {
   	//TODO: eval params to choose a provider
   	val providerMap = getProvider()
   	val provider = providers(providerMap)
     val challenge = provider.returnChallenge()
     val blob = new ByteArrayInputStream(challenge.content)
-    // val token = scala.util.Random.nextInt(100000).toString
+    val insertPstmt = Statements.tlStmts.get.insertPstmt
-    val token = uniqueIntCount.incrementAndGet().toString
+    insertPstmt.setString(1, provider.getId)
-    insertPstmt.synchronized {
+    insertPstmt.setString(2, challenge.secret)
-      insertPstmt.setString(1, token)
+    insertPstmt.setString(3, providerMap)
-      insertPstmt.setString(2, provider.getId)
+    insertPstmt.setString(4, challenge.contentType)
-      insertPstmt.setString(3, challenge.secret)
+    insertPstmt.setBlob(5, blob)
-      insertPstmt.setString(4, providerMap)
+    insertPstmt.executeUpdate()
-      insertPstmt.setString(5, challenge.contentType)
+    val rs: ResultSet = insertPstmt.getGeneratedKeys()
-      insertPstmt.setBlob(6, blob)
+    val token = if(rs.next()){
-      insertPstmt.executeUpdate()
+      rs.getInt("token")
     }
-    token
+    println("Added new challenge: "+ token.toString)
+    token.asInstanceOf[Int]
   }
 
   val task = new Runnable {
   	def run(): Unit = {
+          try {
       val imageNum = stmt.executeQuery("SELECT COUNT(*) AS total FROM challenge")
       var throttleIn = (throttle*1.1).toInt
       if(imageNum.next())
         throttleIn = (throttleIn-imageNum.getInt("total"))
       while(0 < throttleIn){
-        getChallenge(Parameters("","","",Option(Size(0,0))))
+        generateChallenge(Parameters("","","",Option(Size(0,0))))
         throttleIn -= 1
       }
+      
+      val gcStmt = stmt.executeUpdate("DELETE FROM challenge WHERE solved > 10 AND token = (SELECT m.token FROM mapId m, challenge c WHERE c.token = m.token AND m.lastServed = (SELECT MAX(m.lastServed) FROM mapId m, challenge c WHERE c.token=m.token AND DATEDIFF(MINUTE, DATEADD(MINUTE,5,m.lastServed), CURRENT_TIMESTAMP) <= 0))")
+
+    } catch { case e: Exception => println(e) }
   	}
   }
 
@@ -114,63 +134,71 @@ class Captcha(throttle: Int, dbConn: DBConn) {
   }
 
   def getChallenge(param: Parameters): Id = {
-    val idOpt = stmt.synchronized {
+    try {
-      val rs = stmt.executeQuery("SELECT token FROM challenge WHERE solved=FALSE ORDER BY RAND() LIMIT 1")
+      val tokenPstmt = Statements.tlStmts.get.tokenPstmt
-      if(rs.next()) {
+      val rs = tokenPstmt.executeQuery()
-        Some(rs.getString("token"))
+      val tokenOpt = if(rs.next()) {
+        Some(rs.getInt("token"))
       } else {
         None
       }
+      Id(getUUID(tokenOpt.getOrElse(generateChallenge(param))))
+    } catch {case e: Exception => 
+      println(e)
+      Id(getUUID(-1))
     }
-    val id = idOpt.getOrElse(generateChallenge(param))
-    val uuid = getUUID(id)
-    Id(uuid)
   }
 
-  def getUUID(id: String): String = {
+  def getUUID(id: Int): String = {
     val uuid = UUID.randomUUID().toString
-    mapPstmt.synchronized {
+      val mapPstmt = Statements.tlStmts.get.mapPstmt
       mapPstmt.setString(1,uuid)
-      mapPstmt.setString(2,id)
+      mapPstmt.setInt(2,id)
       mapPstmt.executeUpdate()
-    }
     uuid
   }
 
-  def checkAnswer(answer: Answer): Boolean = {
+  def checkAnswer(answer: Answer): String = {
-    val psOpt:Option[ProviderSecret] = selectPstmt.synchronized {
+      val selectPstmt = Statements.tlStmts.get.selectPstmt
       selectPstmt.setString(1, answer.id)
       val rs: ResultSet = selectPstmt.executeQuery()
-      if (rs.first()) {
+      val psOpt = if (rs.first()) {
         val secret = rs.getString("secret")
         val provider = rs.getString("provider")
-        Some(ProviderSecret(provider, secret))
+        val check = providers(provider).checkAnswer(secret, answer.answer)
+        val result = if(check) {
+          val updateSolvedPstmt = Statements.tlStmts.get.updateSolvedPstmt
+          updateSolvedPstmt.setString(1,answer.id)
+          updateSolvedPstmt.executeUpdate()
+          "TRUE"
+        } else {
+          "FALSE"
+        }
+        result
       } else {
-        None
+        "EXPIRED"
       }
-    }
+      psOpt
-    psOpt.map(ps => providers(ps.provider).checkAnswer(ps.secret, answer.answer)).getOrElse(false)
-  }
-
-  def getHash(email: String): Int = {
-    val secret = ""
-    val str = email+secret
-    val hash = str.hashCode()
-    userPstmt.setString(1, email)
-    userPstmt.setInt(2, hash)
-    userPstmt.executeUpdate()
-    hash
   }
 
   def display(): Unit = {
     val rs: ResultSet = stmt.executeQuery("SELECT * FROM challenge")
     println("token\t\tid\t\tsecret\t\tsolved")
     while(rs.next()) {
-      val token = rs.getString("token")
+      val token = rs.getInt("token")
       val id = rs.getString("id")
       val secret = rs.getString("secret")
       val solved = rs.getString("solved")
-      println(s"${token}\t\t${id}\t\t${secret}\t\t${solved}")
+      println(s"${token}\t\t${id}\t\t${secret}\t\t${solved}\n\n")
+    }
+
+    val rss: ResultSet = stmt.executeQuery("SELECT * FROM mapId")
+    println("uuid\t\ttoken\t\tlastServed")
+    while(rss.next()){
+      val uuid = rss.getString("uuid")
+      val token = rss.getInt("token")
+      val lastServed = rss.getTimestamp("lastServed")
+      println(s"${uuid}\t\t${token}\t\t${lastServed}\n\n")
     }
   }
 }
@@ -178,6 +206,7 @@ class Captcha(throttle: Int, dbConn: DBConn) {
 object LCFramework{
   def main(args: scala.Array[String]) {
     val dbConn = new DBConn()
+    Statements.dbConn = dbConn
   	val captcha = new Captcha(2, dbConn)
     val server = new Server(8888, captcha, dbConn)
     captcha.beginThread(2)

src/main/scala/lc/Server.scala

@@ -9,81 +9,19 @@ import lc.HTTPServer._
 
 case class Secret(token: Int)
 
-class RateLimiter(dbConn: DBConn) {
-  private val userLastActive = collection.mutable.Map[Int, Long]()
-  private val userAllowance = collection.mutable.Map[Int, Double]()
-  private val rate = 800000.0
-  private val per = 45.0
-  private val allowance = rate
-
-  private val validatePstmt = dbConn.con.prepareStatement("SELECT hash FROM users WHERE hash = ? LIMIT 1")
-
-  private def validateUser(user: Int) : Boolean = {
-    val allow = if(userLastActive.contains(user)){
-      true
-    } else {
-      validatePstmt.setInt(1, user)
-      val rs = validatePstmt.executeQuery()
-      val validated = if(rs.next()){
-        val hash = rs.getInt("hash")
-        userLastActive(hash) = System.currentTimeMillis()
-        userAllowance(hash) = allowance
-        true
-      } else {
-        false
-      }
-      validated
-    }
-    allow
-  }
-
-  private def checkLimit(user: Int): Boolean = {
-    val current = System.currentTimeMillis()
-    val time_passed = (current - userLastActive(user)) / 1000
-    userLastActive(user) = current
-    userAllowance(user) += time_passed * (rate/per)
-    if(userAllowance(user) > rate){ userAllowance(user) = rate }
-    val allow = if(userAllowance(user) < 1.0){
-      false
-    } else {
-      userAllowance(user) -= 1.0
-      true
-    }
-    allow
-  }
-
-  def checkUserAccess(token: Int) : Boolean = {
-    synchronized {
-      if (validateUser(token)) {
-        return checkLimit(token)
-      } else {
-        return false
-      }
-    }
-  }
-}
-
 class Server(port: Int, captcha: Captcha, dbConn: DBConn){
-  val rateLimiter = new RateLimiter(dbConn)
 	val server = new HTTPServer(port)
 	val host = server.getVirtualHost(null)
 
 	implicit val formats = DefaultFormats
 
 	host.addContext("/v1/captcha",(req, resp) => {
-      val accessToken = Option(req.getHeaders().get("access-token")).map(_.toInt)
+      val body = req.getJson()
-      val access = accessToken.map(rateLimiter.checkUserAccess).getOrElse(false)
+    	val json = parse(body)
-      if(access){
+    	val param = json.extract[Parameters]
-        val body = req.getJson()
+      val id = captcha.getChallenge(param)
-      	val json = parse(body)
+      resp.getHeaders().add("Content-Type","application/json")
-      	val param = json.extract[Parameters]
+      resp.send(200, write(id))
-        val id = captcha.getChallenge(param)
-        resp.getHeaders().add("Content-Type","application/json")
-        resp.send(200, write(id))
-      } else {
-        resp.getHeaders().add("Content-Type","application/json")
-        resp.send(401, write("""{"error": "Not a valid user or rate limit reached!"}"""))
-      }
     	0
     },"POST")
 
@@ -109,21 +47,11 @@ class Server(port: Int, captcha: Captcha, dbConn: DBConn){
     	val answer = json.extract[Answer]
     	val result = captcha.checkAnswer(answer)
     	resp.getHeaders().add("Content-Type","application/json")
-    	val responseContent = if(result) """{"result":"True"}""" else """{"result":"False"}"""
+    	val responseContent = s"""{"result":"$result"}"""
     	resp.send(200,responseContent)
     	0
     },"POST")
 
-    host.addContext("/v1/register", new FileContextHandler(new File("client/")))
-
-    host.addContext("/v1/token", (req,resp) => {
-      val params = req.getParams()
-      val hash = captcha.getHash(params.get("email"))
-      val token = Secret(hash)
-      resp.getHeaders().add("Content-Type", "application/json")
-      resp.send(200, write(token))
-      0
-    })
 
     def start(): Unit = {
       println("Starting server on port:" + port)

tests/locustfile.py

@@ -0,0 +1,44 @@
+from locust import task, between, SequentialTaskSet
+from locust.contrib.fasthttp import FastHttpUser
+import json
+import uuid
+
+class QuickStartUser(SequentialTaskSet):
+    wait_time = between(0.1,1)
+
+    captcha_params = {"level":"some","media":"some","input_type":"some"}
+    answerBody = {"answer": "qwer123"}
+
+    @task
+    def captcha(self):
+        resp = self.client.post(path="/v1/captcha", json=self.captcha_params, name="/captcha")
+        if resp.status_code != 200:
+            print("\nError on /captcha endpoint: ")
+            print(resp)
+            print(resp.text)
+            print("----------------END.C-------------------\n\n")
+        self.answerBody["id"] = json.loads(resp.text).get("id")
+
+    @task
+    def media(self):
+        resp = self.client.get(path="/v1/media?id=%s" % self.answerBody.get("id"), name="/media")
+        if resp.status_code != 200:
+            print("\nError on /media endpoint: ")
+            print(resp)
+            print(resp.text)
+            print("-----------------END.M-------------------\n\n")
+
+    @task
+    def answer(self):
+        resp = self.client.post(path='/v1/answer', json=self.answerBody, name="/answer")
+        if resp.status_code != 200:
+            print("\nError on /answer endpoint: ")
+            print(resp)
+            print(resp.text)
+            print("-------------------END.A---------------\n\n")
+
+
+class User(FastHttpUser):
+    wait_time = between(0.1,1)
+    tasks = [QuickStartUser]
+    host = "http://localhost:8888"