#-----------------------------------------------------------------------------------------------------------------------
# VERSION: Feb 4, 2025
#-----------------------------------------------------------------------------------------------------------------------
import os
import csv
import sys
import glob
import time
import pickle
import pathlib
import urllib.request
#-----------------------------------------------------------------------------------------------------------------------
TARGET_WEBPAGE = None
TARGET_FOLDER = None
DATABASE_NAME = "content.db"
HAS_ANYTHING_CHANGED = False
#-----------------------------------------------------------------------------------------------------------------------
def main():

    global TARGET_WEBPAGE, TARGET_FOLDER

    if (len(sys.argv) != 3):
        print("\nusage: %s <webpage> <folder>" % sys.argv[0])
        print("\nexample: %s http://123.123.123.123:54321/folder1/folder2/ ads_content" % sys.argv[0])
        return

    TARGET_WEBPAGE = sys.argv[1]
    TARGET_FOLDER = sys.argv[2]

    # Check if the directory exists and if not create it

    if (os.path.exists(TARGET_FOLDER) == False):
        os.mkdir(TARGET_FOLDER)

    # Check if the directory is either empty or missing content.db

    is_empty = not os.path.isfile(TARGET_FOLDER + "/" + DATABASE_NAME)

    if (is_empty):
        content_is_empty()
    else:
        content_is_not_empty()

    # Let the caller know whether or not the folder contents has changed in any way

    if (HAS_ANYTHING_CHANGED == True):
        print(TARGET_FOLDER + " has been sync'd")
        sys.exit(5)
    else:
        sys.exit(0)

#-----------------------------------------------------------------------------------------------------------------------
def content_is_empty():

    global TARGET_WEBPAGE, TARGET_FOLDER, HAS_ANYTHING_CHANGED

    print("The folder was empty")

    # Delete any files currently in the content folder

    delete_entire_folder(TARGET_FOLDER)

    # We need to download all of the files, and then create a content.db

    data = get_webpage_list(TARGET_WEBPAGE)

    if (data == None):
        return

    # Download the entire remote folder

    for i in range(0, len(data)):
        path = TARGET_WEBPAGE + data[i][1]
        filename = TARGET_FOLDER + "/" + data[i][1]
        print("Downloading " + filename)
        download_file(path, filename)
        time.sleep(0.100) # delay for 100ms to prevent hammering the web server

    # Create content.db

    with open(TARGET_FOLDER + "/" + DATABASE_NAME, 'wb') as f:
        pickle.dump(data, f)
        f.close()

    HAS_ANYTHING_CHANGED = True

    return

#-----------------------------------------------------------------------------------------------------------------------
def content_is_not_empty():

    # Load up the local content.db

    with open(TARGET_FOLDER + "/" + DATABASE_NAME, 'rb') as f:
        local_data = pickle.load(f)
        f.close()

    # Delete excess files locally that are not in the local database

    delete_excess_local_files(local_data)

    # Verify the local database is correct

    verify_local_database(local_data)

    # Download the remote content metadata

    remote_data = get_webpage_list(TARGET_WEBPAGE)

    if (remote_data == None):
        return

    # Compare the lists

    local_data.sort()
    remote_data.sort()

    if (local_data == remote_data):
        return
    else:
        print("Local folder has differences")
        sync_folder(local_data, remote_data)

    return

#-----------------------------------------------------------------------------------------------------------------------
def sync_folder(local_data, remote_data):

    global HAS_ANYTHING_CHANGED

    # Step 1 - If there is a file in local but not remote, delete it

    i = 0

    while (i < len(local_data)):

        f = local_data[i][1]

        if (filename_isin(f, remote_data) < 0):
            HAS_ANYTHING_CHANGED = True
            print(f + " is not in remote data, deleting the file and database entry")
            # Delete the extra file
            pathlib.Path(TARGET_FOLDER + "/" + f).unlink()
            del local_data[i]
            i -= 1

        i += 1

    # Step 2 - If there is a file in remote but not local, download it

    for i in range(0, len(remote_data)):

        f = remote_data[i][1]

        if (filename_isin(f, local_data) < 0):
            HAS_ANYTHING_CHANGED = True
            print(f + " is not in local data, downloading the file and adding a database entry")
            path = TARGET_WEBPAGE + f
            filename = TARGET_FOLDER + "/" + f
            download_file(path, filename)
            time.sleep(0.100) # delay for 100ms to prevent hammering the web server
            local_data.append(remote_data[i])

    # Step 3 - If there is a file in both, but the date is different, delete the local file and download the remote file

    i = 0

    while (i < len(remote_data)):

        f = remote_data[i][1]

        if ((local_index := filename_isin(f, local_data)) >= 0):

            rt = remote_data[i][0]
            lt = local_data[local_index][0]

            if (rt != lt):
                HAS_ANYTHING_CHANGED = True
                print(f + " timestamp mis-match: " + rt + " VS " + lt)
                # Delete the local file and database entry
                pathlib.Path(TARGET_FOLDER + "/" + f).unlink()
                local_data[local_index][0] = remote_data[i][0]
                path = TARGET_WEBPAGE + f
                filename = TARGET_FOLDER + "/" + f
                download_file(path, filename)
                time.sleep(0.100) # delay for 100ms to prevent hammering the web server

        i += 1

    # Create content.db

    with open(TARGET_FOLDER + "/" + DATABASE_NAME, 'wb') as f:
        pickle.dump(local_data, f)
        f.close()

    return

#-----------------------------------------------------------------------------------------------------------------------
def verify_local_database(local_data):

    global HAS_ANYTHING_CHANGED

    # Check if the local files match the local database

    file_list = glob.glob(TARGET_FOLDER + "/*.*")

    i = 0

    while (i < len(local_data)):

        f = TARGET_FOLDER + "/" + local_data[i][1]

        if (os.path.isfile(f) == False):
            HAS_ANYTHING_CHANGED = True
            print(f + " was not found, deleting the database entry")
            del local_data[i]
            i -= 1

        i += 1

    # Create content.db

    with open(TARGET_FOLDER + "/" + DATABASE_NAME, 'wb') as f:
        pickle.dump(local_data, f)
        f.close()

    return

#-----------------------------------------------------------------------------------------------------------------------
def delete_excess_local_files(local_data):

    global HAS_ANYTHING_CHANGED

    # Check if the local files match the local database

    file_list = glob.glob(TARGET_FOLDER + "/*.*")

    for i in range(0, len(file_list)):

        f = os.path.basename(file_list[i])

        # Don't check on the database itself
        if (f != DATABASE_NAME):
            if (filename_isin(f, local_data) < 0):
                HAS_ANYTHING_CHANGED = True
                print(f + " was not found in the local database, deleting the file")
                # Delete the extra file
                pathlib.Path(file_list[i]).unlink()

    return

#-----------------------------------------------------------------------------------------------------------------------
def parse_file_line(line):

    line = line.split()

    datetime = line[0] + " " + line[1] + line[2]
    filename = line[5].split(">")[1].split("<")[0]

    return datetime + "," + filename

#-----------------------------------------------------------------------------------------------------------------------
def get_webpage_list(url):

    # Download the Website

    try:
        html = urllib.request.urlopen(TARGET_WEBPAGE).read().decode('utf-8')
    except:
        print("ERROR - Unable to open the webpage: " + TARGET_WEBPAGE)
        return None

    # Parse out the list of files

    html = html.split("<br><br>")
    html = html[1].split("</pre>")
    html = html[0].split("<br>")

    del html[-1] # The last line is blank, so delete it

    # Check for an empty Web Page

    if (len(html) == 0):
        print("ERROR - The Web Page '" + TARGET_WEBPAGE + "' has NO files listed on it")
        return

    # Print out the file list to the output

    csvlist = []

    for i in range(0, len(html)):
        csvlist.append(parse_file_line(html[i]))

    data = list(csv.reader(csvlist))

    return data

#-----------------------------------------------------------------------------------------------------------------------
def delete_entire_folder(folder_path):

    [f.unlink() for f in pathlib.Path(folder_path).glob("*") if f.is_file()]

    return

#-----------------------------------------------------------------------------------------------------------------------
def filename_isin(needle, haystack):

    for i in range(0, len(haystack)):
        if (needle == haystack[i][1]):
            return i

    return -1

#-----------------------------------------------------------------------------------------------------------------------
def download_file(path, filename):

    try:
        urllib.request.urlretrieve(path, filename)
    except:
        print("ERROR - Unable to download the file: " + filename)

    return

#-----------------------------------------------------------------------------------------------------------------------
if __name__ == '__main__':

    main()

#-----------------------------------------------------------------------------------------------------------------------