전체 글 (33) 썸네일형 리스트형 [spark dataframe] extract date value using pyspark udf lambda from pyspark.sql import SparkSession import pyspark.sql.functions as func import datetime fnDataReplace = func.udf(lambda s : s.replace('\\','')) fnGetBaseDate = func.udf(lambda value1, s1, s2, s3 : extractBaseDate(value1, s1, s2, s3)) def extractBaseDate(value1, dateCol1, dateCol2, timestampCol): if (dateCol1 is not None) and len(dateCol1) > 13: baseDate = datetime.datetime.strptime(dateCol1, ".. [command] kafka topic CLI # topic list check kafka-topics --bootstrap-server kafka01.host.com:9092,kafka02.host.com:9092,kafka03.host.com:9092 --list # topic details kafka-topics --bootstrap-server kafka01.host.com:9092,kafka02.host.com:9092,kafka03.host.com:9092 --topic topic1 --describe # change topic partition kafka-topics --bootstrap-server kafka01.host.com:9092,kafka02.host.com:9092,kafka03.host.com:9092 --alter --t.. Filebeat to Kafka procedure # ssh ssh id@log_generating_server # cd cd /dir/filebeat_to_kafka # unzip tar -xvf filebeat_to_kafka.tar # Check running processes ps -ef | grep filebeat # stop work kill -9 `ps -ef | grep -w filebeat-big.yml | grep -v grep | awk '{print $2}'` # update yml file vi filebeat-big.yml #-----------------------filebeat prospectors--------------------------------- - input_type: log paths: - /dir/AAA???.. 이전 1 2 3 4 5 ··· 11 다음 목록 더보기