From 5c3bdfeb833f962cbdef7c9b9ffa00f1ff3c294d Mon Sep 17 00:00:00 2001 From: Rahul Rudragoudar Date: Wed, 23 Sep 2020 22:58:42 +0530 Subject: [PATCH] GC, Seed and User management (#52) * Update sql to map uuid to token Signed-off-by: Rahul Rudragoudar * Fix millis to secs conversion Signed-off-by: Rahul Rudragoudar * Add synchronisation to media enpoint DB access Signed-off-by: Rahul Rudragoudar * Change error code for rate limiter Signed-off-by: Rahul Rudragoudar * move prepared statements to Thread Local Storage * Change test end points * init GC * Add GC Signed-off-by: Rahul Rudragoudar * Change status return Signed-off-by: Rahul Rudragoudar * Auto generate token in db Signed-off-by: Rahul Rudragoudar * Remove user management and rate limiting Signed-off-by: Rahul Rudragoudar * Add seed for random number generator Signed-off-by: Rahul Rudragoudar * Store random instance as class member Signed-off-by: Rahul Rudragoudar * Update locustfile Signed-off-by: Rahul Rudragoudar * Add API documentation Signed-off-by: Rahul Rudragoudar * Move updateTimeStamp to getChallenge methdod Remove user tables for the DB Signed-off-by: Rahul Rudragoudar * Update Timestamp when creating mapId entry Signed-off-by: Rahul Rudragoudar * Add request method type Signed-off-by: Rahul Rudragoudar * Minor fixes Signed-off-by: Rahul Rudragoudar --- README.md | 49 ++++++++++ src/main/scala/lc/Main.scala | 157 +++++++++++++++++++-------------- src/main/scala/lc/Server.scala | 86 ++---------------- tests/locustfile.py | 44 +++++++++ 4 files changed, 193 insertions(+), 143 deletions(-) create mode 100644 tests/locustfile.py diff --git a/README.md b/README.md index 008a0d4..5598716 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,55 @@ An image of a word is blurred before being shown to the user. ### LabelCaptcha An image that has a pair of words is created. The answer to one of the words is known and to that of the other is unknown. The user is tested on the known word, and their answer to the unknown word is recorded. If a sufficient number of users agree on their answer to the unknown word, it is transferred to the list of known words. +*** + +## HTTP API +### - `/captcha`: `POST` + - Parameters: + - `level`: `String` - + The difficulty level of a captcha + - easy + - medium + - hard + - `input_type`: `String` - + The type of input option for a captcha + - text + - click (choose) + - `media`: `String` - + The type of media of a captcha + - image + - audio + - gif + - `size`: `dict` - + The dimensions of a captcha (Optional). It needs two more fields nested in this parameter + - `height`: `Int` + - `width`: `Int` + + - Return type: + - `id`: `String` - The uuid of the captcha generated + + +### - `/media`: `POST`,`GET` + - Parameters: + - `id`: `String` - The uuid of the captcha + + - Return type: + - `image`: `Array[Byte]` - The requested media as bytes + + +### - `/answer`: `POST` + - Parameter: + - `id`: `String` - The uuid of the captcha that needs to be solved + - `answer`: `String` - The answer to the captcha that needs to be validated + + - Return Type: + - `result`: `String` - The result after validation/checking of the answer + - True - If the answer is correct + - False - If the answer is incorrect + - Expired - If the time limit to solve the captcha exceeds + +*** + ## Roadmap Things to do in the future: diff --git a/src/main/scala/lc/Main.scala b/src/main/scala/lc/Main.scala index fd5e8ae..1cc71fa 100644 --- a/src/main/scala/lc/Main.scala +++ b/src/main/scala/lc/Main.scala @@ -6,6 +6,8 @@ import java.util.concurrent._ import java.util.UUID import java.sql.{Blob, ResultSet} import java.util.concurrent.atomic.AtomicInteger +import java.io._ +import java.sql.Statement case class Size(height: Int, width: Int) case class Parameters(level: String, media: String, input_type: String, size: Option[Size]) @@ -17,11 +19,11 @@ case class ProviderSecret(provider: String, secret: String) object CaptchaProviders { val providers = Map( "FilterChallenge" -> new FilterChallenge, - "FontFunCaptcha" -> new FontFunCaptcha, + // "FontFunCaptcha" -> new FontFunCaptcha, "GifCaptcha" -> new GifCaptcha, "ShadowTextCaptcha" -> new ShadowTextCaptcha, "RainDropsCaptcha" -> new RainDropsCP, - "LabelCaptcha" -> new LabelCaptcha + // "LabelCaptcha" -> new LabelCaptcha ) def generateChallengeSamples() = { @@ -31,80 +33,98 @@ object CaptchaProviders { } } +class Statements(dbConn: DBConn) { + val insertPstmt = dbConn.con.prepareStatement("INSERT INTO challenge(id, secret, provider, contentType, image) VALUES (?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS ) + val mapPstmt = dbConn.con.prepareStatement("INSERT INTO mapId(uuid, token, lastServed) VALUES (?, ?, CURRENT_TIMESTAMP)") + val selectPstmt = dbConn.con.prepareStatement("SELECT secret, provider FROM challenge WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ? AND DATEDIFF(MINUTE, DATEADD(MINUTE,2,m.lastServed), CURRENT_TIMESTAMP) <= 0)") + val imagePstmt = dbConn.con.prepareStatement("SELECT image FROM challenge c, mapId m WHERE c.token=m.token AND m.uuid = ?") + val updateSolvedPstmt = dbConn.con.prepareStatement("UPDATE challenge SET solved = solved+1 WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ?)") + val tokenPstmt = dbConn.con.prepareStatement("SELECT token FROM challenge WHERE solved < 10 ORDER BY RAND() LIMIT 1") +} + +object Statements { + var dbConn: DBConn = _ + val tlStmts = ThreadLocal.withInitial(() => new Statements(dbConn)) +} + class Captcha(throttle: Int, dbConn: DBConn) { import CaptchaProviders._ private val stmt = dbConn.getStatement() - stmt.execute("CREATE TABLE IF NOT EXISTS challenge(token varchar, id varchar, secret varchar, provider varchar, contentType varchar, image blob, solved boolean default False, PRIMARY KEY(token))") - stmt.execute("CREATE TABLE IF NOT EXISTS mapId(uuid varchar, token varchar, PRIMARY KEY(uuid), FOREIGN KEY(token) REFERENCES challenge(token))") - stmt.execute("CREATE TABLE IF NOT EXISTS users(email varchar, hash int)") + stmt.execute("CREATE TABLE IF NOT EXISTS challenge(token int auto_increment, id varchar, secret varchar, provider varchar, contentType varchar, image blob, solved int default 0, PRIMARY KEY(token))") + stmt.execute("CREATE TABLE IF NOT EXISTS mapId(uuid varchar, token int, lastServed timestamp, PRIMARY KEY(uuid), FOREIGN KEY(token) REFERENCES challenge(token) ON DELETE CASCADE)") - private val insertPstmt = dbConn.con.prepareStatement("INSERT INTO challenge(token, id, secret, provider, contentType, image) VALUES (?, ?, ?, ?, ?, ?)") - private val mapPstmt = dbConn.con.prepareStatement("INSERT INTO mapId(uuid, token) VALUES (?, ?)") - private val selectPstmt = dbConn.con.prepareStatement("SELECT secret, provider FROM challenge WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ?)") - private val imagePstmt = dbConn.con.prepareStatement("SELECT image FROM challenge c, mapId m WHERE c.token=m.token AND m.uuid = ?") - private val updatePstmt = dbConn.con.prepareStatement("UPDATE challenge SET solved = True WHERE token = (SELECT m.token FROM mapId m, challenge c WHERE m.token=c.token AND m.uuid = ?)") - private val userPstmt = dbConn.con.prepareStatement("INSERT INTO users(email, hash) VALUES (?,?)") + private val seed = System.currentTimeMillis.toString.substring(2,6).toInt + private val random = new scala.util.Random(seed) + def getNextRandomInt(max: Int) = random.synchronized { + random.nextInt(max) + } + def getProvider(): String = { - val random = new scala.util.Random val keys = providers.keys - val providerIndex = keys.toVector(random.nextInt(keys.size)) + val providerIndex = keys.toVector(getNextRandomInt(keys.size)) providerIndex } def getCaptcha(id: Id): Array[Byte] = { var image :Array[Byte] = null var blob: Blob = null - val imageOpt = imagePstmt.synchronized { + try { + val imagePstmt = Statements.tlStmts.get.imagePstmt imagePstmt.setString(1, id.id) val rs: ResultSet = imagePstmt.executeQuery() if(rs.next()){ - blob = rs.getBlob("image") - updatePstmt.synchronized{ - updatePstmt.setString(1,id.id) - updatePstmt.executeUpdate() - } - } + blob = rs.getBlob("image") if(blob != null) image = blob.getBytes(1, blob.length().toInt) image } - imageOpt + image + } catch{ case e: Exception => + println(e) + image + } } private val uniqueIntCount = new AtomicInteger() - def generateChallenge(param: Parameters): String = { + def generateChallenge(param: Parameters): Int = { //TODO: eval params to choose a provider val providerMap = getProvider() val provider = providers(providerMap) val challenge = provider.returnChallenge() val blob = new ByteArrayInputStream(challenge.content) - // val token = scala.util.Random.nextInt(100000).toString - val token = uniqueIntCount.incrementAndGet().toString - insertPstmt.synchronized { - insertPstmt.setString(1, token) - insertPstmt.setString(2, provider.getId) - insertPstmt.setString(3, challenge.secret) - insertPstmt.setString(4, providerMap) - insertPstmt.setString(5, challenge.contentType) - insertPstmt.setBlob(6, blob) - insertPstmt.executeUpdate() + val insertPstmt = Statements.tlStmts.get.insertPstmt + insertPstmt.setString(1, provider.getId) + insertPstmt.setString(2, challenge.secret) + insertPstmt.setString(3, providerMap) + insertPstmt.setString(4, challenge.contentType) + insertPstmt.setBlob(5, blob) + insertPstmt.executeUpdate() + val rs: ResultSet = insertPstmt.getGeneratedKeys() + val token = if(rs.next()){ + rs.getInt("token") } - token + println("Added new challenge: "+ token.toString) + token.asInstanceOf[Int] } val task = new Runnable { def run(): Unit = { + try { val imageNum = stmt.executeQuery("SELECT COUNT(*) AS total FROM challenge") var throttleIn = (throttle*1.1).toInt if(imageNum.next()) throttleIn = (throttleIn-imageNum.getInt("total")) while(0 < throttleIn){ - getChallenge(Parameters("","","",Option(Size(0,0)))) + generateChallenge(Parameters("","","",Option(Size(0,0)))) throttleIn -= 1 } + + val gcStmt = stmt.executeUpdate("DELETE FROM challenge WHERE solved > 10 AND token = (SELECT m.token FROM mapId m, challenge c WHERE c.token = m.token AND m.lastServed = (SELECT MAX(m.lastServed) FROM mapId m, challenge c WHERE c.token=m.token AND DATEDIFF(MINUTE, DATEADD(MINUTE,5,m.lastServed), CURRENT_TIMESTAMP) <= 0))") + + } catch { case e: Exception => println(e) } } } @@ -114,63 +134,71 @@ class Captcha(throttle: Int, dbConn: DBConn) { } def getChallenge(param: Parameters): Id = { - val idOpt = stmt.synchronized { - val rs = stmt.executeQuery("SELECT token FROM challenge WHERE solved=FALSE ORDER BY RAND() LIMIT 1") - if(rs.next()) { - Some(rs.getString("token")) + try { + val tokenPstmt = Statements.tlStmts.get.tokenPstmt + val rs = tokenPstmt.executeQuery() + val tokenOpt = if(rs.next()) { + Some(rs.getInt("token")) } else { None } + Id(getUUID(tokenOpt.getOrElse(generateChallenge(param)))) + } catch {case e: Exception => + println(e) + Id(getUUID(-1)) } - val id = idOpt.getOrElse(generateChallenge(param)) - val uuid = getUUID(id) - Id(uuid) } - def getUUID(id: String): String = { + def getUUID(id: Int): String = { val uuid = UUID.randomUUID().toString - mapPstmt.synchronized { + val mapPstmt = Statements.tlStmts.get.mapPstmt mapPstmt.setString(1,uuid) - mapPstmt.setString(2,id) + mapPstmt.setInt(2,id) mapPstmt.executeUpdate() - } uuid } - def checkAnswer(answer: Answer): Boolean = { - val psOpt:Option[ProviderSecret] = selectPstmt.synchronized { + def checkAnswer(answer: Answer): String = { + val selectPstmt = Statements.tlStmts.get.selectPstmt selectPstmt.setString(1, answer.id) val rs: ResultSet = selectPstmt.executeQuery() - if (rs.first()) { + val psOpt = if (rs.first()) { val secret = rs.getString("secret") val provider = rs.getString("provider") - Some(ProviderSecret(provider, secret)) + val check = providers(provider).checkAnswer(secret, answer.answer) + val result = if(check) { + val updateSolvedPstmt = Statements.tlStmts.get.updateSolvedPstmt + updateSolvedPstmt.setString(1,answer.id) + updateSolvedPstmt.executeUpdate() + "TRUE" + } else { + "FALSE" + } + result } else { - None + "EXPIRED" } - } - psOpt.map(ps => providers(ps.provider).checkAnswer(ps.secret, answer.answer)).getOrElse(false) - } - - def getHash(email: String): Int = { - val secret = "" - val str = email+secret - val hash = str.hashCode() - userPstmt.setString(1, email) - userPstmt.setInt(2, hash) - userPstmt.executeUpdate() - hash + psOpt } def display(): Unit = { val rs: ResultSet = stmt.executeQuery("SELECT * FROM challenge") println("token\t\tid\t\tsecret\t\tsolved") while(rs.next()) { - val token = rs.getString("token") + val token = rs.getInt("token") val id = rs.getString("id") val secret = rs.getString("secret") val solved = rs.getString("solved") - println(s"${token}\t\t${id}\t\t${secret}\t\t${solved}") + println(s"${token}\t\t${id}\t\t${secret}\t\t${solved}\n\n") + } + + val rss: ResultSet = stmt.executeQuery("SELECT * FROM mapId") + println("uuid\t\ttoken\t\tlastServed") + while(rss.next()){ + val uuid = rss.getString("uuid") + val token = rss.getInt("token") + val lastServed = rss.getTimestamp("lastServed") + println(s"${uuid}\t\t${token}\t\t${lastServed}\n\n") } } } @@ -178,6 +206,7 @@ class Captcha(throttle: Int, dbConn: DBConn) { object LCFramework{ def main(args: scala.Array[String]) { val dbConn = new DBConn() + Statements.dbConn = dbConn val captcha = new Captcha(2, dbConn) val server = new Server(8888, captcha, dbConn) captcha.beginThread(2) diff --git a/src/main/scala/lc/Server.scala b/src/main/scala/lc/Server.scala index 3120325..863a368 100644 --- a/src/main/scala/lc/Server.scala +++ b/src/main/scala/lc/Server.scala @@ -9,81 +9,19 @@ import lc.HTTPServer._ case class Secret(token: Int) -class RateLimiter(dbConn: DBConn) { - private val userLastActive = collection.mutable.Map[Int, Long]() - private val userAllowance = collection.mutable.Map[Int, Double]() - private val rate = 800000.0 - private val per = 45.0 - private val allowance = rate - - private val validatePstmt = dbConn.con.prepareStatement("SELECT hash FROM users WHERE hash = ? LIMIT 1") - - private def validateUser(user: Int) : Boolean = { - val allow = if(userLastActive.contains(user)){ - true - } else { - validatePstmt.setInt(1, user) - val rs = validatePstmt.executeQuery() - val validated = if(rs.next()){ - val hash = rs.getInt("hash") - userLastActive(hash) = System.currentTimeMillis() - userAllowance(hash) = allowance - true - } else { - false - } - validated - } - allow - } - - private def checkLimit(user: Int): Boolean = { - val current = System.currentTimeMillis() - val time_passed = (current - userLastActive(user)) / 1000 - userLastActive(user) = current - userAllowance(user) += time_passed * (rate/per) - if(userAllowance(user) > rate){ userAllowance(user) = rate } - val allow = if(userAllowance(user) < 1.0){ - false - } else { - userAllowance(user) -= 1.0 - true - } - allow - } - - def checkUserAccess(token: Int) : Boolean = { - synchronized { - if (validateUser(token)) { - return checkLimit(token) - } else { - return false - } - } - } -} - class Server(port: Int, captcha: Captcha, dbConn: DBConn){ - val rateLimiter = new RateLimiter(dbConn) val server = new HTTPServer(port) val host = server.getVirtualHost(null) implicit val formats = DefaultFormats host.addContext("/v1/captcha",(req, resp) => { - val accessToken = Option(req.getHeaders().get("access-token")).map(_.toInt) - val access = accessToken.map(rateLimiter.checkUserAccess).getOrElse(false) - if(access){ - val body = req.getJson() - val json = parse(body) - val param = json.extract[Parameters] - val id = captcha.getChallenge(param) - resp.getHeaders().add("Content-Type","application/json") - resp.send(200, write(id)) - } else { - resp.getHeaders().add("Content-Type","application/json") - resp.send(401, write("""{"error": "Not a valid user or rate limit reached!"}""")) - } + val body = req.getJson() + val json = parse(body) + val param = json.extract[Parameters] + val id = captcha.getChallenge(param) + resp.getHeaders().add("Content-Type","application/json") + resp.send(200, write(id)) 0 },"POST") @@ -109,21 +47,11 @@ class Server(port: Int, captcha: Captcha, dbConn: DBConn){ val answer = json.extract[Answer] val result = captcha.checkAnswer(answer) resp.getHeaders().add("Content-Type","application/json") - val responseContent = if(result) """{"result":"True"}""" else """{"result":"False"}""" + val responseContent = s"""{"result":"$result"}""" resp.send(200,responseContent) 0 },"POST") - host.addContext("/v1/register", new FileContextHandler(new File("client/"))) - - host.addContext("/v1/token", (req,resp) => { - val params = req.getParams() - val hash = captcha.getHash(params.get("email")) - val token = Secret(hash) - resp.getHeaders().add("Content-Type", "application/json") - resp.send(200, write(token)) - 0 - }) def start(): Unit = { println("Starting server on port:" + port) diff --git a/tests/locustfile.py b/tests/locustfile.py new file mode 100644 index 0000000..9cf8fa9 --- /dev/null +++ b/tests/locustfile.py @@ -0,0 +1,44 @@ +from locust import task, between, SequentialTaskSet +from locust.contrib.fasthttp import FastHttpUser +import json +import uuid + +class QuickStartUser(SequentialTaskSet): + wait_time = between(0.1,1) + + captcha_params = {"level":"some","media":"some","input_type":"some"} + answerBody = {"answer": "qwer123"} + + @task + def captcha(self): + resp = self.client.post(path="/v1/captcha", json=self.captcha_params, name="/captcha") + if resp.status_code != 200: + print("\nError on /captcha endpoint: ") + print(resp) + print(resp.text) + print("----------------END.C-------------------\n\n") + self.answerBody["id"] = json.loads(resp.text).get("id") + + @task + def media(self): + resp = self.client.get(path="/v1/media?id=%s" % self.answerBody.get("id"), name="/media") + if resp.status_code != 200: + print("\nError on /media endpoint: ") + print(resp) + print(resp.text) + print("-----------------END.M-------------------\n\n") + + @task + def answer(self): + resp = self.client.post(path='/v1/answer', json=self.answerBody, name="/answer") + if resp.status_code != 200: + print("\nError on /answer endpoint: ") + print(resp) + print(resp.text) + print("-------------------END.A---------------\n\n") + + +class User(FastHttpUser): + wait_time = between(0.1,1) + tasks = [QuickStartUser] + host = "http://localhost:8888"