[Jsoup] GET & POST crawling

Example

    private fun token(email: String, password: String): String {
        val url = "https://asdf.com/tokens"

        val headers = mapOf(
            "Accept" to "application/json",
            "Content-Type" to "application/json",
            "Authorization" to "null"
        )
        val json = mapOf(
            "auth_type" to "CREDENTIAL",
            "credential_type_payload" to mapOf(
                "email" to email,
                "password" to password,
            ),
        ).toJson()

        return CrawlUtil.post(url, headers, json)["token"] as String
    }

---

CrawlUtil

import org.jsoup.Connection
import org.jsoup.Jsoup

object CrawlUtil {
    private const val TIMEOUT = 5 * 60 * 1000
    private const val MAX_BODY_SIZE = 0
    private const val IGNORE_CONTENT_TYPE = true

    private fun connection(
        url: String,
        headers: Map<String, String>,
        json: String,
        data: Map<String, String>? = emptyMap()
    ): Connection {
        var connection = Jsoup.connect(url)

        headers.forEach {
            connection = connection.header(it.key, it.value)
        }

        data!!.forEach {
            connection = connection.data(it.key, it.value)
        }

        return connection
            .timeout(TIMEOUT)
            .maxBodySize(MAX_BODY_SIZE)
            .ignoreContentType(IGNORE_CONTENT_TYPE)
            .requestBody(json)
    }

    fun get(
        url: String,
        headers: Map<String, String>,
        json: String,
        data: Map<String, String>? = emptyMap()
    ): Map<String, Any> {
        return connection(url, headers, json, data)
            .get()
            .text()
            .toObject()
    }

    fun post(
        url: String,
        headers: Map<String, String>,
        json: String,
        data: Map<String, String>? = emptyMap()
    ): Map<String, Any> {
        return connection(url, headers, json, data)
            .post()
            .text()
            .toObject()
    }
}

ExtensionUtil

inline fun <reified T> String.toObject(): T {
    return JacksonUtil.toObject(this)
}

JacksonUtil

import com.fasterxml.jackson.core.type.TypeReference
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer
import com.fasterxml.jackson.module.afterburner.AfterburnerModule
import com.fasterxml.jackson.module.kotlin.KotlinModule
import java.time.LocalDate
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter

object JacksonUtil {
    val objectMapper: ObjectMapper = ObjectMapper()
        .registerModule(KotlinModule.Builder().build())
        .registerModule(JavaTimeModule())
        .registerModule(AfterburnerModule())
        .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false)
        .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
        .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
        .registerModule(
            JavaTimeModule().apply {
                addDeserializer(LocalDate::class.java, LocalDateDeserializer(DateTimeFormatter.ISO_DATE))
                addDeserializer(
                    LocalDateTime::class.java,
                    LocalDateTimeDeserializer(DateTimeFormatter.ISO_DATE_TIME)
                )
            }
        )

    fun toJson(obj: Any?): String {
        if (obj == null) {
            return ""
        }

        return objectMapper.writeValueAsString(obj)
    }

    fun merged(a: MutableMap<String, Any>, b: Any): Map<String, Any> {
        return objectMapper
            .readerForUpdating(a)
            .readValue(toJson(b))
    }

    inline fun <reified T> toObject(s: String): T {
        return objectMapper.readValue(s, object : TypeReference<T>() {})
    }
}

https://stackoverflow.com/questions/27463036/jsoup-http-post-with-payload

Jsoup HTTP POST with payload

https://partnerjun.tistory.com/43

Java HTML parser, Jsoup로 원하는 값 얻어내기 - 로그인

https://stackoverflow.com/questions/10012399/how-to-post-data-into-website-using-jsoup

How to POST Data into website using Jsoup

https://dirask.com/posts/Java-Jsoup-FIX-for-exception-com-fasterxml-jackson-core-io-JsonEOFException-Unexpected-end-of-input-in-field-name-at-Source-java-io-StringReader-1725dc0f-line-1-column-1046939-kDLGL1

Java Jsoup - FIX for exception com.fasterxml.jackson.core.io.JsonEOFException: Unexpected end-of-input in field name at [Source:

https://yj-something.tistory.com/2

JAVA 웹 크롤링(Web crawling)