[spark] dropTempView

fun drop(table: String) {
    val spark = SparkUtil.instance(CommonUtil.methodName())

    spark.catalog().dropTempView(table)
}

object SparkUtil {
    fun instance(name: String? = ""): SparkSession {
        return make(name!!)
    }

    private fun make(name: String): SparkSession {
        return config(
            SparkSession.builder().appName(name)
        ).enableHiveSupport().orCreate
    }

    private fun config(builder: SparkSession.Builder): SparkSession.Builder {
        val map = YamlUtils.read(this::class.java, "spark", Extension.YAML)

        var b = builder
        map.keys.forEach {
            val k = it
            val v = map[k]

            b = when (v) {
                is Long -> b.config(k, v)
                is String -> b.config(k, v)
                is Double -> b.config(k, v)
                is Boolean -> b.config(k, v)
                else -> b
            }
        }

        return b
    }
}

object YamlUtils {
    fun <T> read(clazz: Class<T>, name: String, extension: Extension): Map<String, Any> {
        val readValue = CommonUtil.flatMap(
            Yaml().load(
                FileUtil.inputStream(clazz, name, extension)
            )
        ).toMutableMap()

        readValue.forEach {
            if (EnvUtil.contains(it.value)) {
                readValue[it.key] = EnvUtil.convert(it.value)
            }
        }

        return readValue.toMap()
    }
}

object EnvUtil {
    private val REGEX = Regex("\\$\\{?([a-zA-Z_/]*)}?")

    fun convert(value: Any): String {
        var v = value.toString()

        val map = find(v)
        map.forEach {
            v = v.replace(it.key, get(it.value) ?: it.key)
        }

        return v
    }

    fun contains(value: Any): Boolean {
        return find(value).isNotEmpty()
    }

    private fun find(value: Any): MutableMap<String, String> {
        val map = mutableMapOf<String, String>()

        val matches = REGEX.findAll(value.toString())
        matches.forEach {
            map[it.groupValues[0]] = it.groupValues[1]
        }

        return map
    }

    fun get(key: String): String? {
        return System.getenv(key)
    }
}

object CommonUtil {
    inline fun methodName(): String = object {}::class.java.enclosingMethod.name
    inline fun className(): String = object {}::class.java.enclosingClass.simpleName

    inline fun <reified T> setDefault(s: String, defaultVal: T): T {
        return if (StringUtils.isEmpty(s) || StringUtils.equalsIgnoreCase(s, "null")) {
            defaultVal
        } else {
            @Suppress("UNCHECKED_CAST") s as T
        }
    }

    inline fun <reified T> setDefault(args: Array<String>, index: Int, defaultVal: T): T {
        return try {
            setDefault(args[index], defaultVal)
        } catch (expected: Exception) {
            defaultVal
        }
    }

    operator fun <T> get(list: List<T>, i: Int): T? {
        return try {
            list[i]
        } catch (expected: Exception) {
            null
        }
    }

    fun flatMap(
        map: Map<String, Any>,
        flat: MutableMap<String, Any>? = mutableMapOf(),
        prefix: String? = null
    ): Map<String, Any> {
        flat!!

        for (k in map.keys) {
            val key = if (prefix == null) {
                k
            } else {
                "$prefix.$k"
            }

            if (map[k] is Map<*, *>) {
                val m = map[k] as Map<String, Any>

                flatMap(m, flat, key)
                continue
            }

            flat[key] = map[k]!!
        }

        return flat
    }
}

spark:
  sql:
    crossJoin.enabled: true
    hive.convertMetastoreOrc: false
    sources.partitionOverwriteMode: dynamic
    broadcastTimeout: 36000
    autoBroadcastJoinThreshold: -1
    legacy:
      parquet:
        int96RebaseModeInRead: CORRECTED
        int96RebaseModeInWrite: CORRECTED
        datetimeRebaseModeInRead: CORRECTED
        datetimeRebaseModeInWrite: CORRECTED
      timeParserPolicy: LEGACY
      storeAssignmentPolicy: LEGACY

hive:
  exec:
    dynamic.partition: true
    dynamic.partition.mode: nonstrict
  metastore:
    client:
      factory:
        class: com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory

https://www.tabnine.com/code/java/methods/org.apache.spark.sql.catalog.Catalog/dropTempView

org.apache.spark.sql.catalog.Catalog.dropTempView java code examples | Tabnine