Upload folder using huggingface_hub

README.md

@@ -0,0 +1,7 @@
+
+
+这里对于5个特征分开，并对时间序列进行平滑，确保每10分钟都有一个数据点。
+
+Total removed duplicates: 1926788
+Total added smooth data: 6641770
+Total final rows: 14093265

preprocess.py

@@ -0,0 +1,201 @@
+import os
+import re
+import pandas as pd
+from tqdm import tqdm
+from datetime import datetime, timedelta
+
+# 从分地点csv转换为分变量csv
+def process_csv_files(input_dir, output_dir):
+    """
+    遍历指定目录下的所有.csv文件，并按要求处理数据。
+    :param input_dir: 输入目录，包含原始.csv文件
+    :param output_dir: 输出目录，保存处理后的.csv文件
+    """
+    # 定义关键词列表
+    KEYWORDS = ["表面位移", "表面裂缝", "深部位移", "温度", "雨量"]
+    # 遍历输入目录中的所有文件
+    for file_name in os.listdir(input_dir):
+        # if file_name == "辰溪孝坪镇江东村山体滑坡.csv":
+        if file_name.endswith(".csv"):  # 确保是.csv文件
+            # 获取文件名前4个字作为A
+            A = file_name[:4]
+            print(f"Processing file: {file_name}, A = {A}")
+
+            # 读取CSV文件
+            file_path = os.path.join(input_dir, file_name)
+            df = pd.read_csv(file_path)
+
+            # 遍历每个关键词
+            for keyword in KEYWORDS:
+                print(f"processing {keyword}")
+                # 创建一个空的字典，用于存储每个设备名称对应的数据
+                device_data_dict = {}
+                device_counter = 0  # 用于记录每个关键词下的设备名称编号
+
+                # 遍历每一行数据
+                for index, row in tqdm(df.iterrows(), total=len(df), desc=f"file_name={file_name}, keyword={keyword}"):
+                    # 检查设备名称是否包含当前关键词
+                    if re.search(keyword, row["设备名称"]):
+                        # 如果设备名称包含关键词，提取设备名称、时间、采集值x、y、z
+                        device_name = row["设备名称"]
+                        data = row[["时间", "采集值x", "采集值y", "采集值z"]]
+
+                        # 如果设备名称第一次出现，分配一个编号
+                        if device_name not in device_data_dict:
+                            device_data_dict[device_name] = {"data": [], "id": device_counter}
+                            device_counter += 1
+
+                        # 获取设备名称的编号
+                        device_id = device_data_dict[device_name]["id"]
+
+                        # 追加数据到对应设备名称的列表中
+                        device_data_dict[device_name]["data"].append(data)
+
+                # 保存每个设备名称编号对应的数据为新的CSV文件
+                for device_name, info in device_data_dict.items():
+                    device_id = info["id"]
+                    data_list = info["data"]
+
+                    # 将数据列表转换为DataFrame
+                    device_df = pd.DataFrame(data_list, columns=["时间", "采集值x", "采集值y", "采集值z"])
+
+                    # 确保输出目录存在
+                    keyword_output_dir = os.path.join(output_dir, keyword)
+                    os.makedirs(keyword_output_dir, exist_ok=True)
+
+                    # 保存为新的CSV文件
+                    output_file_name = f"{keyword}_{device_id}_{A}.csv"
+                    output_file_path = os.path.join(keyword_output_dir, output_file_name)
+                    device_df.to_csv(output_file_path, index=False)
+                    print(f"Saved file: {output_file_path}")
+
+
+# 函数1：删除完全相同的重复记录
+def remove_duplicates(data_list):
+    seen = set()
+    result = []
+    duplicate_count = 0  # 用于统计删除的重复记录数
+    for item in data_list:
+        if item not in seen:
+            seen.add(item)
+            result.append(item)
+        else:
+            duplicate_count += 1
+    print(f"origin_len:{len(data_list)}")
+    print(f"after_duplication_len:{len(result)}")
+    print(f"Removed duplicates: {duplicate_count}")
+    return result, duplicate_count
+
+
+# 函数2：对时间不连续的部分进行平滑过渡填充
+def smooth_data(data_list):
+    if not data_list or len(data_list) < 2:
+        return data_list, 0
+
+    # 解析时间戳和值
+    def parse_timestamp_and_value(record):
+        parts = record.strip().split(',')
+        timestamp_str = parts[0]
+        values = [float(x) if x != 'NaN' and x != '' else 0.0 for x in parts[1:]]  # 如果是NaN，转换为0
+        timestamp = datetime.fromisoformat(timestamp_str.replace('+08', '+0800'))
+        return timestamp, values
+
+    # 格式化为字符串
+    def format_record(timestamp, values):
+        timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M:%S%z').replace('+0800', '+08')
+        values_str = ','.join(f"{v:.10f}" for v in values)  # 使用通用格式化，保留足够的精度
+        return f"{timestamp_str},{values_str}\n"
+
+    header = data_list[0]  # 提取表头
+    data_list = data_list[1:]
+    result = [header]
+    supply_count = 0  # 用于统计添加的平滑数据数
+    for i in tqdm(range(len(data_list) - 1), desc="Processing data", unit="step"):
+        current_timestamp, current_values = parse_timestamp_and_value(data_list[i])
+        next_timestamp, next_values = parse_timestamp_and_value(data_list[i + 1])
+        # 确保 current_values 和 next_values 的长度一致
+        if len(current_values) != len(next_values):
+            raise ValueError(f"数据行 {i} 和 {i+1} 的列数不一致")
+        result.append(format_record(current_timestamp, current_values))
+        # 如果时间差超过10分钟，进行平滑过渡填充
+        time_diff = (next_timestamp - current_timestamp).total_seconds() / 60
+        if time_diff > 10:
+            steps = int(time_diff / 10)
+            supply_count += steps - 1
+            value_steps = [(next_values[j] - current_values[j]) / steps for j in range(len(current_values))]
+            for step in range(1, steps):
+                new_timestamp = current_timestamp + timedelta(minutes=step * 10)
+                new_values = [current_values[j] + step * value_steps[j] for j in range(len(current_values))]
+                result.append(format_record(new_timestamp, new_values))
+    # 添加最后一个数据点
+    last_timestamp, last_values = parse_timestamp_and_value(data_list[-1])
+    result.append(format_record(last_timestamp, last_values))
+    print(f"supply_num={supply_count}")
+    return result, supply_count
+
+
+# 对单个文件：读取文件、调用处理函数、写回文件
+def dep_and_smooth(input_file, output_file):
+    try:
+        # 读取文件内容
+        with open(input_file, 'r') as file:
+            data_list = file.readlines()
+        # 删除重复记录
+        print("Removing duplicates...")
+        data_list, duplicate_count = remove_duplicates(data_list)
+        # 对时间不连续的部分进行平滑过渡填充
+        print("Smoothing data...")
+        data_list, supply_count = smooth_data(data_list)
+        # 写回文件
+        with open(output_file, 'w') as file:
+            file.writelines(data_list)
+        print(f"处理完成，结果已写入 {output_file}")
+        return duplicate_count, supply_count, len(data_list)
+    except Exception as e:
+        print(f"处理过程中发生错误：{e}")
+        return 0, 0, 0
+
+
+# 函数4：遍历文件夹并处理所有文件
+def process_folder(input_folder, output_folder):
+    total_duplicate_count = 0
+    total_supply_count = 0
+    total_final_row_count = 0
+
+    # 确保输出文件夹存在
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+
+    # 遍历输入文件夹
+    for root, dirs, files in os.walk(input_folder):
+        for file in files:
+            if file.endswith('.csv'):
+                # 构建输入文件路径
+                input_file_path = os.path.join(root, file)
+                # 构建输出文件路径
+                relative_path = os.path.relpath(root, input_folder)
+                output_subfolder = os.path.join(output_folder, relative_path)
+                if not os.path.exists(output_subfolder):
+                    os.makedirs(output_subfolder)
+                output_file_path = os.path.join(output_subfolder, file)
+                # 处理文件
+                print(f"Processing file: {input_file_path}")
+                duplicate_count, supply_count, final_row_count = dep_and_smooth(input_file_path, output_file_path)
+                total_duplicate_count += duplicate_count
+                total_supply_count += supply_count
+                total_final_row_count += final_row_count
+
+    print(f"Total removed duplicates: {total_duplicate_count}")
+    print(f"Total added smooth data: {total_supply_count}")
+    print(f"Total final rows: {total_final_row_count}")
+
+
+if __name__ == "__main__":
+    process_folder(
+        input_folder="/home/mby/time-series-transformer-demo/datasets/category_data",
+        output_folder="/home/mby/time-series-transformer-demo/datasets/category_data_processed"
+    )
+
+    # dep_and_smooth(
+    #     input_file="/home/mby/time-series-transformer-demo/datasets/category_data/表面裂缝/表面裂缝_0_辰溪孝坪.csv",
+    #     output_file="/home/mby/time-series-transformer-demo/datasets/category_data/out.csv")

preprocess2.py

@@ -0,0 +1,22 @@
+import os
+import pandas as pd
+
+def save_csv_files(folder_path):
+    # 遍历文件夹及其子文件夹
+    for root, dirs, files in os.walk(folder_path):
+        for filename in files:
+            # 检查文件是否为CSV文件
+            if filename.endswith(".csv"):
+                file_path = os.path.join(root, filename)
+                try:
+                    # 读取CSV文件
+                    df = pd.read_csv(file_path)
+                    # 重新保存CSV文件（覆盖原文件）
+                    df.to_csv(file_path, index=False, encoding="utf-8-sig")
+                    print(f"Re-saved file: {file_path}")
+                except Exception as e:
+                    print(f"Error processing file {file_path}: {e}")
+
+# 指定要处理的文件夹路径
+folder_path = input("请输入要处理的文件夹路径：")
+save_csv_files(folder_path)