tour.py

import collections
import os
import os.path
import re
import shutil
import socket
import subprocess
import sys
import uuid
from contextlib import closing
from pathlib import Path
from string import Template
from typing import Dict, List, Optional, Tuple
from urllib.parse import urlparse

from h2o_wave import Q, app, main, ui

example_dir = os.path.dirname(os.path.realpath(__file__))
tour_tmp_dir = os.path.join(example_dir, '_tour_apps_tmp')

_base_url = os.environ.get('H2O_WAVE_BASE_URL', '/')
_app_address = urlparse(os.environ.get('H2O_WAVE_APP_ADDRESS', 'http://127.0.0.1:8000'))
default_example_name = 'hello_world'
vsc_extension_path = os.path.join(example_dir, '..', '..', 'tools', 'vscode-extension')


def scan_free_port(port: int):
    while True:
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
            if sock.connect_ex(('localhost', port)):
                return port
        port += 1


class Example:
    def __init__(self, filename: str, title: str, description: str, source: str):
        self.name = os.path.splitext(filename)[0]
        self.filename = filename
        self.title = title
        self.description = description
        self.source = source
        self.previous_example: Optional[Example] = None
        self.next_example: Optional[Example] = None
        self.process: Optional[subprocess.Popen] = None

    def start(self, filename: str, is_app: bool, q: Q):
        env = os.environ.copy()
        env['H2O_WAVE_BASE_URL'] = _base_url
        env['H2O_WAVE_ADDRESS'] = os.environ.get('H2O_WAVE_ADDRESS', 'http://127.0.0.1:10101')
        # The environment passed into Popen must include SYSTEMROOT, otherwise Popen will fail when called
        # inside python during initialization if %PATH% is configured, but without %SYSTEMROOT%.
        if sys.platform.lower().startswith('win'):
            env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
        if is_app:
            q.app.app_port = scan_free_port(q.app.app_port)
            env['H2O_WAVE_APP_ADDRESS'] = f'http://{_app_address.hostname}:{q.app.app_port}'
            self.process = subprocess.Popen([
                sys.executable, '-m', 'uvicorn',
                '--host', '0.0.0.0',
                '--port', str(q.app.app_port),
                f'examples.{filename.replace(".py", "")}:main',
            ], env=env)
        else:
            self.process = subprocess.Popen([sys.executable, os.path.join(example_dir, filename)], env=env)

    def stop(self):
        if self.process and self.process.returncode is None:
            self.process.terminate()
            self.process.wait()


def read_lines(p: str) -> List[str]:
    with open(p, encoding='utf-8') as f:
        return f.readlines()


def read_file(p: str) -> str:
    with open(p, encoding='utf-8') as f:
        return f.read()


def strip_comment(line: str) -> str:
    """Returns the content of a line without '#' and ' ' characters

    remove leading '#', but preserve '#' that is part of a tag
    example:
    >>> '# #hello '.strip('#').strip()
    '#hello'
    """
    return line.strip('#').strip()


def parse_tags(description: str) -> Tuple[str, List[str]]:
    """Creates tags from description.

    Accepts a description containing tags and returns a (new_description, tags) tuple.

    The convention for tags:
    1. Any valid twitter hashtag

    For example, accept a description in any of the following forms

    1. Use a checklist to group a set of related checkboxes. #form #checkbox #checklist

    2. Use a checklist to group a set of related checkboxes.
       #form #checkbox #checklist

    3. Use a #checklist to group a set of related checkboxes.
       #form #checkbox

    and return
    ('Use a checklist to group a set of related checkboxes.', ['checkbox', 'checklist', 'form']). The list of tags will
    be sorted and all tags will be converted to lowercase.

    Args:
        description: Complete description of an example.
    Returns:
        A tuple of new_description and a sorted list of tags. new_description is created by removing the '#' characters
        from the description.
    """
    hashtag_regex_pattern = r"(\s+)#(\w*[a-zA-Z]+\w*)\b"
    pattern = re.compile(hashtag_regex_pattern)
    matches = pattern.findall(' ' + description)

    # Retrieve tags from the matches
    tags = sorted(list(set([x[-1].lower() for x in matches])))

    # Remove the '#' before the tags in description
    new_d = pattern.sub(r'\1\2', ' ' + description)

    # Remove the last line in description if it has only tags
    *lines, last_line = new_d.strip().splitlines()
    last_line_has_tags_only = len(last_line.strip()) > 1 and all([x.strip().lower() in tags for x in last_line.split()])
    if last_line_has_tags_only:
        # Return all lines except the last line
        return '\n'.join(lines), tags

    # Remove the last sentence if it has only tags
    *sentences, last_sentence = last_line.split('. ')
    last_sentence_has_tags_only = len(last_sentence.strip()) > 1 and all(
        [x.strip().lower() in tags for x in last_sentence.split()])
    if last_sentence_has_tags_only:
        # Return all lines and all sentences in the last line except the last sentence
        lines.extend(sentences)
        return '\n'.join(lines) + '.', tags

    # Return the complete description
    lines.append(last_line)
    return '\n'.join(lines), tags


def load_example(filename: str) -> Example:
    contents = read_file(os.path.join(example_dir, filename))
    parts = contents.split('---', maxsplit=1)
    header, source = parts[0].strip().splitlines(), parts[1].strip()
    title, description = strip_comment(header[0]), [strip_comment(x) for x in header[1:]]
    new_description, _ = parse_tags('\n'.join(description))
    return Example(filename, title, new_description, source)


def load_examples(filenames: List[str]) -> Dict[str, Example]:
    examples = collections.OrderedDict()
    for filename in filenames:
        example = load_example(filename)
        examples[example.name] = example
    example_list = [e for e in examples.values()]
    k = len(example_list) - 1
    for i, e in enumerate(example_list):
        if i > 0:
            e.previous_example = example_list[i - 1]
        if i < k:
            e.next_example = example_list[i + 1]
    return examples


app_title = 'H2O Wave Tour'
header_height = 76
blurb_height = 56
mobile_blurb_height = 76


async def setup_page(q: Q):
    py_content = ''
    parser_path = os.path.join(example_dir, 'tour_autocomplete_parser.py')
    utils_path = os.path.join(example_dir, 'tour_autocomplete_utils.py')
    # In prod.
    if os.path.exists(parser_path) and os.path.exists(utils_path):
        with open(parser_path, 'r') as f:
            py_content = f.read()
        with open(utils_path, 'r') as f:
            py_content += f.read()
    # When run in development from Wave repo.
    elif os.path.exists(vsc_extension_path):
        with open(os.path.join(vsc_extension_path, 'server', 'parser.py'), 'r') as f:
            py_content = f.read()
        with open(os.path.join(vsc_extension_path, 'server', 'utils.py'), 'r') as f:
            py_content += f.read()

    if py_content:
        py_content += '''
def get_wave_completions(line, character, file_content):
    completion_type, leaf_val = get_completion_type(line, character, file_content)
    if completion_type in ['args', 'events', 'zones', 'client', 'app', 'user']:
        completion_items = []
        file_metadata = fill_completion(file_content)
        if completion_type == 'events' and leaf_val:
            completion_items = list(getattr(file_metadata, completion_type).get(leaf_val, []))
        elif completion_type == 'events' and leaf_val is None:
            completion_items = list(getattr(file_metadata, completion_type).keys())
        elif leaf_val is None:
            completion_items = getattr(file_metadata, completion_type)
        return [{'label': label, 'kind': 6, 'sort_text': '0'} for label in completion_items]
    elif completion_type == 'themes':
        return [{'label': theme, 'kind': 13, 'sort_text': '0'} for theme in themes]
    elif completion_type == 'icons':
        return [{'label': icon, 'kind': 13, 'sort_text': '0'} for icon in fluent_icons]
        '''
    js_code = ''
    with open(os.path.join(example_dir, 'tour.js'), 'r') as f:
        js_code = f.read()
    template = Template(js_code).substitute(
        tour_assets=q.app.tour_assets,
        base_url=_base_url,
        snippets1=q.app.snippets1,
        snippets2=q.app.snippets2,
        py_content=py_content
    )
    q.page['meta'] = ui.meta_card(
        box='',
        title=app_title,
        scripts=[ui.script(q.app.tour_assets + '/loader.min.js')],
        script=ui.inline_script(content=template, requires=['require'], targets=['monaco-editor']),
        layouts=[
            ui.layout(
                breakpoint='xs',
                zones=[
                    ui.zone('mobile_header'),
                    ui.zone('main',
                            zones=[
                                ui.zone('code', size=f'calc(50vh - {(header_height + mobile_blurb_height) / 2}px)'),
                                ui.zone('preview', size=f'calc(50vh - {(header_height + mobile_blurb_height) / 2}px)'),
                            ]),
                    ui.zone('mobile_blurb')
                ],
            ),
            ui.layout(breakpoint='m', zones=[
                ui.zone('header'),
                ui.zone('blurb'),
                ui.zone('main', size=f'calc(100vh - {header_height + blurb_height}px)', direction=ui.ZoneDirection.ROW,
                        zones=[
                            ui.zone('code'),
                            ui.zone('preview')
                        ])
            ]),
        ])
    nav_links = [
        ('docs', 'Wave docs', 'https://wave.h2o.ai/docs/getting-started'),
        ('discussions', 'Discussions', 'https://github.com/h2oai/wave/discussions'),
        ('blog', 'Blog', 'https://wave.h2o.ai/blog'),
        ('cloud', 'H2O AI Cloud', 'https://h2o.ai/platform/ai-cloud/'),
        ('h2o', 'H2O', 'https://www.h2o.ai/'),
    ]
    q.page['header'] = ui.header_card(
        box='header',
        title=app_title,
        subtitle=f'{len(catalog)} Interactive Examples',
        image=f'{q.app.tour_assets}/h2o-logo.svg',
        items=[
            ui.links(inline=True, items=[ui.link(label=link[1], path=link[2], target='_blank') for link in nav_links])
        ])
    q.page['mobile_header'] = ui.header_card(
        box='mobile_header',
        title=app_title,
        subtitle=f'{len(catalog)} Interactive Examples',
        image=f'{q.app.tour_assets}/h2o-logo.svg',
        nav=[
            ui.nav_group('Links', items=[ui.nav_item(name=link[0], label=link[1], path=link[2]) for link in nav_links])
        ])
    q.page['blurb'] = ui.section_card(box='blurb', title='', subtitle='', items=[])
    q.page['mobile_blurb'] = ui.form_card(box='mobile_blurb', items=[])
    q.page['code'] = ui.markup_card(
        box='code',
        title='',
        content='<div id="monaco-editor" style="position: absolute; top: 45px; bottom: 15px; right: 15px; left: 2px"/>'
    )
    # Put tmp placeholder <div></div> to simulate blank screen.
    q.page['preview'] = ui.frame_card(box='preview', title='Preview', content='<div></div>')
    await q.page.save()


def make_blurb(q: Q):
    example = q.client.active_example
    blurb_card = q.page['blurb']
    blurb_card.title = example.title
    blurb_card.subtitle = example.description
    # HACK: Recreate dropdown every time (by dynamic name) to control value (needed for next / prev btn functionality).
    items = [ui.dropdown(name=q.args['#'] or default_example_name, width='300px', value=example.name, trigger=True,
                         choices=[ui.choice(name=e.name, label=e.title) for e in catalog.values()])]
    if example.previous_example:
        items.append(ui.button(name=f'#{example.previous_example.name}', label='Prev'))
    if example.next_example:
        items.append(ui.button(name=f'#{example.next_example.name}', label='Next', primary=True))
    blurb_card.items = items
    q.page['mobile_blurb'].items = [ui.inline(direction='row', justify='center', items=items)]


async def show_example(q: Q, example: Example):
    # Clear demo page
    demo_page = q.site[f'/{q.client.path}']
    demo_page.drop()
    await demo_page.save()

    filename = os.path.join(tour_tmp_dir, f'{q.client.path}.py')
    code = q.events.editor.change if q.events.editor else example.source
    code = code.replace("`", "\\`")
    is_app = code.find('@app(') > 0
    with open(filename, 'w') as f:
        fixed_path = code
        if is_app:
            fixed_path = fixed_path.replace("@app('/demo')", f"@app('/{q.client.path}')")
        else:
            fixed_path = fixed_path.replace("site['/demo']", f"site['/{q.client.path}']")
        f.write(fixed_path)
    if is_app:
        filename = '.'.join([tour_tmp_dir, f'{q.client.path}.py']).split(os.sep)[-1]

    # Stop active example, if any.
    active_example = q.client.active_example
    if active_example:
        active_example.stop()

    # Start new example
    example.start(filename, is_app, q)
    q.client.active_example = example

    # Update example blurb
    make_blurb(q)

    # Update preview title
    q.page['preview'].title = f'Preview of {example.filename}'
    q.page['code'].title = example.filename
    await q.page.save()

    if q.client.is_first_load:
        # Make sure all the JS has loaded properly.
        await q.sleep(1)
        q.client.is_first_load = False

    # Update code display
    if not q.events.editor:
        code = code.replace('$', '\\$')
        q.page['meta'].script = ui.inline_script(f'editor.setValue(`{code}`)', requires=['editor'])
        await q.page.save()
        if q.args['#']:
            q.page['meta'].script = ui.inline_script('editor.setScrollPosition({ scrollTop: 0 }); editor.focus()',
                                                     requires=['editor'])

    # HACK
    # The ?e= appended to the path forces the frame to reload.
    # The url param is not actually used.
    q.page['preview'].path = f'{_base_url}{q.client.path}?e={example.name}'
    await q.page.save()


async def on_startup():
    # Clean up previous tmp dir.
    await on_shutdown()
    os.mkdir(tour_tmp_dir)
    shutil.copyfile(os.path.join(example_dir, 'synth.py'), os.path.join(tour_tmp_dir, 'synth.py'))
    shutil.copyfile(os.path.join(example_dir, 'plot_d3.js'), os.path.join(tour_tmp_dir, 'plot_d3.js'))


async def on_shutdown():
    dirpath = Path(tour_tmp_dir)
    if dirpath.exists():
        shutil.rmtree(dirpath)


@app('/tour', on_startup=on_startup, on_shutdown=on_shutdown)
async def serve(q: Q):
    if not q.app.initialized:
        q.app.app_port = 10102
        q.app.tour_assets, = await q.site.upload_dir(os.path.join(example_dir, 'tour-assets'))
        base_snippets_path = os.path.join(example_dir, 'base-snippets.json')
        component_snippets_path = os.path.join(example_dir, 'component-snippets.json')
        # Prod.
        if os.path.exists(base_snippets_path) and os.path.exists(component_snippets_path):
            q.app.snippets1, q.app.snippets2, = await q.site.upload([base_snippets_path, component_snippets_path])
        # When run in development from Wave repo.
        elif os.path.exists(vsc_extension_path):
            q.app.snippets1, q.app.snippets2, = await q.site.upload([
                os.path.join(vsc_extension_path, 'base-snippets.json'),
                os.path.join(vsc_extension_path, 'component-snippets.json')
            ])
        q.app.initialized = True
    if not q.client.initialized:
        q.client.initialized = True
        q.client.is_first_load = True
        q.client.path = uuid.uuid4()
        await setup_page(q)

    search = q.args[q.args['#'] or default_example_name]
    if search and not q.events.editor:
        q.page['meta'] = ui.meta_card(box='', redirect=f'#{search}')

    await show_example(q, catalog[q.args['#'] or default_example_name])


example_filenames = [line.strip() for line in read_lines(os.path.join(example_dir, 'tour.conf')) if
                     not line.strip().startswith('#')]
catalog = load_examples(example_filenames)
print('----------------------------------------')
print(' Welcome to the H2O Wave Interactive Tour!')
print('')
print(' Go to http://localhost:10101/tour')
print('----------------------------------------')