The Goal

A simple capture system that:

• Grabs any highlighted text instantly

• Optionally includes screenshots

• Works with Obsidian's Local REST API

• Integrates with Linux desktop shortcuts

The Solution

A Python script that leverages the Local REST API plugin for Obsidian. The plugin exposes a REST API that lets you read, create, and modify notes programmatically.

How It Works

1. Text Selection: Uses xclip (X11) or wl-paste (Wayland) to read the primary selection—text you've highlighted but haven't copied

2. Screenshots: Integrates with Flameshot (X11) or grim/slurp (Wayland) for area selection

3. REST API: Uploads images and appends content to your vault via HTTP

4. Notifications: Desktop feedback via notify-send

The Script

#!/usr/bin/env python3
"""
Obsidian Quick Capture Script

Captures highlighted text (and optionally screenshots) to an Obsidian note
via the Local REST API plugin.

Usage:
    obs_capture.py           # Capture selected text only
    obs_capture.py -s        # Capture text + screenshot
    obs_capture.py -n Note   # Use custom target note
"""

import argparse
import subprocess
import requests
import datetime
import os
import sys
import urllib3

# Suppress insecure HTTPS request warnings (expected for Local REST API self-signed certs)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# --- Configuration ---
API_KEY = "YOUR_API_KEY_HERE"  # Get from Obsidian Local REST API plugin settings
BASE_URL = "https://127.0.0.1:27124"

# Default paths (relative to vault root)
DEFAULT_NOTE = "00-Inbox/Quick Captures.md"
ATTACHMENT_DIR = "Attachments/"

HEADERS = {
    "Authorization": f"Bearer {API_KEY}"
}


def notify(title: str, message: str, urgency: str = "normal"):
    """Send a desktop notification using notify-send."""
    try:
        subprocess.run(
            ["notify-send", "-u", urgency, title, message],
            check=False
        )
    except FileNotFoundError:
        print(f"[{urgency.upper()}] {title}: {message}")


def get_selected_text() -> str:
    """Grab currently highlighted text from primary selection (X11 or Wayland)."""
    # Try xclip first (X11 primary selection)
    try:
        result = subprocess.run(
            ["xclip", "-o", "-selection", "primary"],
            capture_output=True,
            text=True,
            check=True
        )
        return result.stdout.strip()
    except (subprocess.CalledProcessError, FileNotFoundError):
        pass

    # Fallback to wl-paste (Wayland)
    try:
        result = subprocess.run(
            ["wl-paste", "-p"],
            capture_output=True,
            text=True,
            check=True
        )
        return result.stdout.strip()
    except (subprocess.CalledProcessError, FileNotFoundError):
        pass

    return ""


def take_screenshot(filepath: str) -> bool:
    """Allow user to draw a rectangle and capture it (X11 with Flameshot or Wayland with grim/slurp)."""
    # Try Flameshot first (X11)
    try:
        subprocess.run(
            ["flameshot", "gui", "-p", filepath],
            check=True,
            capture_output=True
        )
        if os.path.exists(filepath):
            return True
    except (subprocess.CalledProcessError, FileNotFoundError):
        pass

    # Fallback to grim/slurp (Wayland)
    try:
        subprocess.run(
            f'grim -g "$(slurp)" "{filepath}"',
            shell=True,
            check=True
        )
        return os.path.exists(filepath)
    except subprocess.CalledProcessError:
        return False

    return False


def check_api_connection() -> bool:
    """Verify the Obsidian Local REST API is reachable."""
    try:
        response = requests.get(
            f"{BASE_URL}/",
            headers=HEADERS,
            verify=False,
            timeout=5
        )
        return response.status_code in (200, 404)
    except requests.exceptions.RequestException:
        return False


def ensure_note_exists(note_path: str) -> bool:
    """Create the target note if it doesn't exist."""
    url = f"{BASE_URL}/vault/{note_path}"
    try:
        response = requests.get(url, headers=HEADERS, verify=False)
        if response.status_code == 200:
            return True

        if response.status_code == 404:
            create_response = requests.put(
                url,
                headers={**HEADERS, "Content-Type": "text/markdown"},
                data="# Quick Captures\n".encode("utf-8"),
                verify=False
            )
            return create_response.status_code in (200, 201, 204)
        return False
    except requests.exceptions.RequestException:
        return False


def upload_image(img_path: str, dest_filename: str) -> bool:
    """Upload an image to the Obsidian vault."""
    url = f"{BASE_URL}/vault/{ATTACHMENT_DIR}{dest_filename}"

    try:
        with open(img_path, "rb") as f:
            img_data = f.read()

        response = requests.put(
            url,
            headers={**HEADERS, "Content-Type": "image/png"},
            data=img_data,
            verify=False
        )
        return response.status_code in (200, 201, 204)
    except requests.exceptions.RequestException:
        return False


def append_to_note(note_path: str, content: str) -> bool:
    """Append content to a note in the vault."""
    url = f"{BASE_URL}/vault/{note_path}"

    try:
        response = requests.post(
            url,
            headers={**HEADERS, "Content-Type": "text/markdown"},
            data=content.encode("utf-8"),
            verify=False
        )
        return response.status_code in (200, 201, 204)
    except requests.exceptions.RequestException:
        return False


def parse_args() -> argparse.Namespace:
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(
        description="Capture highlighted text (and optionally screenshots) to Obsidian.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s              Capture selected text only
  %(prog)s -s           Capture text + screenshot
  %(prog)s -n Inbox.md  Use custom target note
"""
    )
    parser.add_argument(
        "-s", "--screenshot",
        action="store_true",
        help="Also capture a screenshot (text is always captured)"
    )
    parser.add_argument(
        "-n", "--note",
        default=DEFAULT_NOTE,
        help=f"Target note path (default: {DEFAULT_NOTE})"
    )
    return parser.parse_args()


def main():
    args = parse_args()
    timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    # 1. Check API connection
    if not check_api_connection():
        notify(
            "Obsidian Capture Failed",
            "Cannot connect to Obsidian Local REST API.\nMake sure Obsidian is running with the plugin enabled.",
            "critical"
        )
        sys.exit(1)

    # 2. Get selected text
    text = get_selected_text()
    if not text:
        notify(
            "Obsidian Capture Failed",
            "No text selected. Highlight some text first.",
            "critical"
        )
        sys.exit(1)

    # 3. Ensure target note exists
    if not ensure_note_exists(args.note):
        notify(
            "Obsidian Capture Failed",
            f"Could not create or access note: {args.note}",
            "critical"
        )
        sys.exit(1)

    # 4. Build content to append
    content_to_append = f"\n\n---\n\n**{timestamp}**\n\n"
    content_to_append += f"> {text}\n"

    # 5. Handle screenshot if requested
    img_filename = None
    if args.screenshot:
        img_filename = f"capture_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
        img_filepath = f"/tmp/{img_filename}"

        if take_screenshot(img_filepath):
            if upload_image(img_filepath, img_filename):
                content_to_append += f"\n![[{img_filename}]]\n"
            else:
                notify("Warning", "Screenshot taken but failed to upload to Obsidian.")
            if os.path.exists(img_filepath):
                os.remove(img_filepath)
        else:
            notify("Warning", "Screenshot cancelled or failed. Text will still be captured.")

    # 6. Append to note
    if append_to_note(args.note, content_to_append):
        preview = text[:50] + "..." if len(text) > 50 else text
        msg = f'Captured: "{preview}"'
        if img_filename:
            msg += " + screenshot"
        notify("Obsidian Capture", msg)
    else:
        notify("Obsidian Capture Failed", "Failed to append content to note.", "critical")
        sys.exit(1)


if __name__ == "__main__":
    main()

Setting Up Keyboard Shortcuts

On GNOME, register global hotkeys via gsettings:

# Text capture: Super+Shift+C
gsettings set org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom0/ name 'Obsidian Capture Text'
gsettings set org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom0/ command '/home/you/.local/bin/obs_capture.py'
gsettings set org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom0/ binding '<Super><Shift>c'

# Text + screenshot: Super+Shift+S
gsettings set org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom1/ name 'Obsidian Capture Screenshot'
gsettings set org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom1/ command '/home/you/.local/bin/obs_capture.py -s'
gsettings set org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom1/ binding '<Super><Shift>s'

# Register them
gsettings set org.gnome.settings-daemon.plugins.media-keys custom-keybindings "['/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom0/', '/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom1/']"

The Result

Highlight any text, press Super+Shift+C, and it appears in your vault:

---

**2026-02-22 21:34:19**

> Your captured text here

With -s flag or Super+Shift+S, you get text plus an embedded screenshot.

Requirements

• Obsidian with Local REST API plugin enabled

• Python 3 + requests library

• X11: xclip, flameshot, libnotify-bin

• Wayland: wl-clipboard, grim, slurp

# Ubuntu/Debian (X11)
sudo apt install xclip flameshot libnotify-bin
pip install requests

Why This Matters

The best capture system is the one you actually use. By reducing friction to a single hotkey, I find myself capturing more—interesting quotes, code snippets, error messages—without breaking my workflow. Everything flows into a single inbox note that I process later.

The full documentation is available at ~/.local/share/doc/obs_capture.md.

Happy capturing!

If you are an Obsidian enthusiast like me and youtube is your one of the source for learning, I have a good news for you. I have developed plugin with chrome extension that transforms YouTube videos content into four type of structured notes using advanced AI analysis. Whether you're creating educational content, research notes, or learning materials, this plugin automatically extracts key insights, generates summaries, and creates step-by-step tutorials from YouTube videos.

• Executive Summary - best for quick overview and decision making with 250 words, key insights, main points and recommendations

• Comprehensive Summary - useful for learning and implementation with detailed 8K words and actionable steps

• Brief Summary - Quick summaries with essential points and key takeaways with 100 words

• Custom Summary - based on your custom prompts for your needs

I'm happy to share my comprehensive compilation of commands for the Claude Code CLI. This guide I hope will help you navigate and utilize the Claude Code CLI's full potential, whether you're automating tasks, managing permissions, or working across multiple directories. With this, you'll be able to handle everything from basic commands to advanced workflows, ensuring you can optimize your use of Claude Code CLI for any project.

Key Highlights:

🎯 Most Useful Flags

• -p, --print - Headless mode (perfect for scripts/CI)

• -y, --dangerously-skip-permissions - YOLO mode

• --model - Switch between Sonnet/Opus

• --add-dir - Work across multiple directories

• --output-format json - Parse responses programmatically

• --max-turns N - Control cost and prevent runaway agents

• --permission-mode plan - Review before execution

🔥 Power Combos

For Automation:

claude -p --output-format json --max-turns 3 "task"

For Safe Refactoring:

claude --permission-mode plan --allowedTools "Read" "Edit" "task"

For Complex Multi-repo Tasks:

claude -y --model opus --add-dir ../backend ../frontend "task"

📋 Interactive Commands

• /help, /undo, /retry, /model, /history

• @filename, @folder/, @url, @git mentions

🛡️ Permission Control

• --allowedTools - Whitelist safe operations

• --disallowedTools - Blacklist dangerous commands

• --permission-mode - Choose approval level

The artifact above has everything organized with examples for CI/CD, pre-commit hooks, batch processing, and more!

Basic Commands

Essential Flags

🚀 Automation & Output

⚙️ Model & Behavior

📁 Working Directories

🔐 Permissions & Safety

🔄 Session Management

Interactive Slash Commands

Use these inside the Claude Code REPL:

@ Mentions

Reference context in your prompts:

Practical Examples

1. Headless Mode (CI/CD, Scripts)

# Simple one-shot
claude -p "run tests and fix failures"

# With JSON output for parsing
claude -p "analyze code" --output-format json

# Streaming JSON
claude -p "refactor" --output-format stream-json

2. YOLO Mode (Auto-approve everything)

# Skip all confirmations
claude -y "migrate from Jest to Vitest"

# Or full flag
claude --dangerously-skip-permissions "rebuild the API"

3. Custom Models

# Use Opus
claude --model opus "complex refactoring task"

# Use specific version
claude --model claude-sonnet-4.5-20250929 "task"

4. Multi-directory Projects

# Work across multiple dirs
claude --add-dir ../backend ../frontend "sync API types"

5. Permission Control

# Auto-approve safe operations
claude --allowedTools "Read" "Bash(git log:*)" "task"

# Block dangerous operations
claude --disallowedTools "Bash(rm *)" "Bash(sudo *)" "task"

# Plan mode (review before executing)
claude --permission-mode plan "refactor database"

6. Session Management

# Continue last conversation
claude -c

# Continue with new prompt
claude -c -p "now add error handling"

# Resume specific session
claude -r "abc123" "continue the migration"

7. Piped Input

# Analyze logs
cat error.log | claude -p "find the root cause"

# Process command output
git diff | claude -p "write commit message"

# Multiple files
cat *.py | claude -p "find security issues"

8. Custom Instructions

# Add system-level instructions
claude --append-system-prompt "Use TypeScript strict mode" "build API"

# Combine with other flags
claude -y --append-system-prompt "Write tests for everything" "task"

9. Limit Turns (Cost Control)

# Max 3 back-and-forth turns
claude -p --max-turns 3 "fix the bug"

# Prevent runaway agents
claude -y --max-turns 5 "complex task"

10. Verbose Debugging

# See detailed logs
claude --verbose "task"

# Combine with headless
claude -p --verbose "task" 2> debug.log

Advanced Workflows

Pre-commit Hook

#!/bin/bash
# .git/hooks/pre-commit

changed_files=$(git diff --cached --name-only)
echo "$changed_files" | claude -p --max-turns 1 \
  "Review these changes for issues" --output-format json

CI/CD Integration

# In GitHub Actions
- name: Code Review
  run: |
    git diff main..HEAD | claude -p \
      --output-format json \
      "Review changes and suggest improvements"

Automated Refactoring

# Safe refactoring with plan mode
claude --permission-mode plan \
  --allowedTools "Read" "Edit" \
  "refactor all components to use hooks"

Batch Processing

# Process multiple files
for file in src/*.ts; do
  claude -p "add JSDoc to $file" --max-turns 1
done

Configuration Tips

Environment Variables

# Set default model
export CLAUDE_CODE_MODEL="claude-sonnet-4.5-20250929"

# Set API key
export ANTHROPIC_API_KEY="your-key"

# Set base URL (for proxies)
export ANTHROPIC_BASE_URL="http://localhost:8000"

Create Aliases

# Add to ~/.bashrc or ~/.zshrc
alias cc="claude"
alias ccp="claude -p"
alias ccy="claude -y"
alias ccplan="claude --permission-mode plan"
alias ccverbose="claude --verbose"

Permission Modes Explained

Tool Patterns for allowedTools/disallowedTools

# Git operations
"Bash(git log:*)" "Bash(git diff:*)" "Bash(git status:*)"

# File operations
"Read" "Edit" "Write"

# Safe commands
"Bash(ls:*)" "Bash(cat:*)" "Bash(grep:*)"

# Package managers
"Bash(npm *)" "Bash(pip *)" "Bash(cargo *)"

# Dangerous (for disallowedTools)
"Bash(rm *)" "Bash(sudo *)" "Bash(chmod *)"

Cost Optimization

# Limit turns to control cost
claude -p --max-turns 2 "task"

# Use Sonnet for most tasks (cheaper than Opus)
claude --model sonnet "task"

# Headless mode for automation (no interactive overhead)
claude -p "task"

Troubleshooting

Check Version

claude --version

View Logs

# Enable verbose logging
claude --verbose "task" 2>&1 | tee debug.log

Test Connection

# Simple test
claude -p "echo hello"

Clear Cache

# Remove session data
rm -rf ~/.claude-code/sessions/

Quick Reference Card

Most Used Flags:

-p              # Headless mode
-y              # Skip confirmations
-c              # Continue conversation
--model opus    # Use Opus model
--add-dir       # Add directories
--verbose       # Debug logging
--max-turns N   # Limit turns

Permission Control:

--permission-mode plan          # Review first
--allowedTools "Read" "Edit"    # Whitelist
--disallowedTools "Bash(rm *)"  # Blacklist

Output Formats:

--output-format text         # Human readable
--output-format json         # Structured
--output-format stream-json  # Streaming

Resources

• Official Docs: https://docs.claude.com/en/docs/claude-code

• Best Practices: https://www.anthropic.com/engineering/claude-code-best-practices

• GitHub: https://github.com/anthropics/claude-code

Pro Tip: Combine flags for powerful workflows:

claude -y --model opus --add-dir ../shared --max-turns 5 \
  --allowedTools "Read" "Edit" "Bash(git *)" \
  "migrate entire codebase to TypeScript"

To start building the chat app, you need to obtain your OpenAI API key. The API key authenticates your requests to the OpenAI API. Not having the API key will prevent you from accessing the OpenAI API. You won't be able to utilize its features. After creating an account on the OpenAI website, you can obtain the API key from there.

Step 1: Sign up for an account on the OpenAI website to access your API key.

Step 2: Create your project directory. Its structure should look like this:

project
├── .env
├── .gitignore
├── app.py
├── requirements.txt
└── README.md

Step 3: copy and paste to the requirements.txt. Run in terminal pip install -r requirements.txt

aiohappyeyeballs==2.3.5
aiohttp==3.10.3
aiosignal==1.3.1
altair==5.4.0
annotated-types==0.7.0
anyio==4.4.0
attrs==24.2.0
blinker==1.8.2
cachetools==5.4.0
certifi==2024.7.4
charset-normalizer==3.3.2
click==8.1.7
distro==1.9.0
frozenlist==1.4.1
gitdb==4.0.11
GitPython==3.1.43
h11==0.14.0
httpcore==1.0.5
httpx==0.27.0
idna==3.7
Jinja2==3.1.4
jiter==0.5.0
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
markdown-it-py==3.0.0
MarkupSafe==2.1.5
mdurl==0.1.2
multidict==6.0.5
narwhals==1.3.0
numpy==2.0.1
openai==1.40.3
packaging==24.1
pandas==2.2.2
pillow==10.4.0
protobuf==5.27.3
pyarrow==17.0.0
pydantic==2.8.2
pydantic_core==2.20.1
pydeck==0.9.1
Pygments==2.18.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.1
referencing==0.35.1
requests==2.32.3
rich==13.7.1
rpds-py==0.20.0
six==1.16.0
smmap==5.0.1
sniffio==1.3.1
streamlit==1.37.1
tenacity==8.5.0
toml==0.10.2
tornado==6.4.1
tqdm==4.66.5
typing_extensions==4.12.2
tzdata==2024.1
urllib3==2.2.2
watchdog==4.0.2
yarl==1.9.4

Step 4: Open the app.py file in a text editor like VS Code and start to write or copy the code below into it.

• Model Options: Place the models you want to use. At present, GPT-4o-mini is efficient and cheap.

• Persona: This is a system message that will directly affect the way our chat app responds to our prompt. So, it should be carefully crafted.

import streamlit as st
import sqlite3
import openai
from openai import OpenAI
import time
import os

# Constants
MODEL_OPTIONS = ["gpt-4o-mini", "gpt-4o", "gpt-3-5"]
PERSONAS_OPTIONS = {
    "Analytical": "Provide detailed, logical analyses.",
    "Business_Consultant": "Offer strategic business advice and insights.",
    "Chef": "Share cooking tips, recipes, and culinary advice.",
    "Code_Reviewer": "Analyze code snippets for best practices and potential bugs.",
    "Concise": "Give brief, to-the-point responses.",
    "Creative": "Offer imaginative and original responses.",
    "Default": "Act as a helpful assistant.",  # Default persona
}
TONE_OPTIONS = [
                "Professional", 
                "Casual", 
                "Friendly", 
                "Formal", 
                "Humorous"
            ]

DEFAULT_MODEL = "gpt-4o-mini"
DEFAULT_PERSONA = "Default"

Step 5: Export the API key to the system environment so that our app can safely access the key.

# Initialize OpenAI client
client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

Step 6: Now, let's set up the SQLite3 database. After setting up the database, we need to set up the cache. The cache is a temporary storage location for data that is frequently accessed. It speeds up the processing of user input.

# Database setup
def init_db():
    conn = sqlite3.connect('chat_history.db')
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS messages
                 (role TEXT, content TEXT, timestamp REAL)''')
    conn.commit()
    return conn

st.set_page_config(page_title="Chatbee🐝", page_icon="🐝")

# Cache setup
@st.cache_data(ttl=3600)
def get_openai_response(messages, model, max_tokens, temperature):
    response = client.chat.completions.create(
        model=model,
        messages=messages,
        max_tokens=max_tokens,
        temperature=temperature
    )
    return response.choices[0].message.content

# Process user input
def process_user_input(prompt):
    # Add user message to chat history
    st.session_state.messages.append({"role": "user", "content": prompt})
    with st.chat_message("user"):
        st.markdown(prompt)

    # Prepare messages for API call
    messages = [
        {"role": "system", "content": f"You are acting as a {persona_key} persona. {persona}\n\nTone: {tone}"},
        *st.session_state.messages
    ]

    # Get AI response
    with st.chat_message("assistant"):
        message_placeholder = st.empty()
        full_response = ""
        response = get_openai_response(messages, model, max_tokens, temperature)
        full_response += response
        message_placeholder.markdown(full_response + "▌")
        message_placeholder.markdown(full_response)

    # Add AI response to chat history
    st.session_state.messages.append({"role": "assistant", "content": full_response})

    # Save to database
    conn = init_db()
    c = conn.cursor()
    c.executemany("INSERT INTO messages VALUES (?, ?, ?)", [
        ("user", prompt, time.time()),
        ("assistant", full_response, time.time())
    ])
    conn.commit()
    conn.close()

Step 7: All the configuration and parameters for this application are placed in the sidebar. We can select the model, persona, and tone that we want to use. We can also adjust the advanced settings, such as the maximum number of tokens and the temperature.

# Sidebar configuration
with st.sidebar:
    st.markdown("<h3 style='text-align: center;'>⚙️ Configurations 🔧</h3>", unsafe_allow_html=True)

    model = st.sidebar.selectbox("Model", MODEL_OPTIONS, index=MODEL_OPTIONS.index(DEFAULT_MODEL))
    persona_key = st.sidebar.selectbox("Persona", list(PERSONAS_OPTIONS.keys()), index=list(PERSONAS_OPTIONS.keys()).index(DEFAULT_PERSONA))
    persona = PERSONAS_OPTIONS[persona_key]
    tone = st.sidebar.selectbox("Tone", TONE_OPTIONS)

    with st.sidebar.expander("Advanced Settings"):
        max_tokens = st.slider("Max Tokens", min_value=50, max_value=2000, value=150, step=50)
        temperature = st.slider("Temperature", min_value=0.0, max_value=1.0, value=0.7, step=0.1)

    st.sidebar.markdown("---")  # Add a separator
    if st.sidebar.button("Clear Chat History"):
        st.session_state.messages = []
        st.rerun()

#

Step 9: The main chat window displays the chat history between we and the AI assistant. We can type a message and send it to the AI assistant, which will respond with a message that is tailored to our needs.

# Main chat window
st.markdown('<h1 style="text-align: center; color: #6ca395;">Chatbee🐝</h1>', unsafe_allow_html=True)
st.markdown('<p style="text-align: center; color: #FF0000;">always at your service</p>', unsafe_allow_html=True)

# Initialize session state
if 'messages' not in st.session_state:
    st.session_state.messages = []

# Display chat history
for message in st.session_state.messages:
    with st.chat_message(message["role"]):
        st.markdown(message["content"])

# Chat input
if prompt := st.chat_input("Hello, how can I help you?"):
    process_user_input(prompt)

Complete code is available for cloning: https://github.com/emeeran/Chatbee.git

Step 1: Install Required Packages

• Open a terminal on your Ubuntu system and run the following command:

sudo apt-get update && sudo apt-get install cifs-utils

Step 2: Create a Mount Point

• Create a directory to mount your Synology NAS:

sudo mkdir /media/synology

Step 3: Get NAS IP Address

• Find the IP address of your Synology NAS. You can do this by:

  • Type in the browser: https://finds.synology.com/

  • Logging into your Synology NAS web interface and looking for the IP address under

  • "Control Panel" > "Network" > "LAN"

  • Using the Synology Assistant software (if installed)

  • Checking your router's DHCP client list

Step 4: Mount the NAS

• Mount your Synology NAS using the following command:

sudo mount -t cifs //NAS_IP_ADDRESS/share_name /media/synology -o username=your_username,password=your_password

Replace:

• NAS_IP_ADDRESS with the IP address of your Synology NAS

• share_name with the name of the shared folder on your NAS

• your_username and your_password with your Synology NAS login credentials

Step 5: Use a credentials file

Instead of passing the username and password directly in the command, create a credentials file and reference it in the mount command:

sudo nano /etc/cifs-credentials.txt

Add the following lines to the file:

username=Meeran
password=qvt****-YMQ8bpm!qrq

Then, mount the NAS using:

sudo mount -t cifs //192.168.1.100/home /media/synology -o credentials=/etc/cifs-credentials.txt

Step 6: Open the fstab file

Run the following command in a terminal:

Bash

sudo nano /etc/fstab

Step 7: Add the mount entry.

Add the following line at the end of the file:

//192.168.1.222/home /media/synology cifs credentials=/etc/cifs-credentials.txt,iocharset=utf8,vers=3.0 0 0

Replace:

• //192.168.1.222/home with the UNC path to your Synology NAS share

• /media/synology with the mount point on your Ubuntu system

• /etc/cifs-credentials.txt with the path to your credentials file.

Step 8: Save and exit

Save the changes and exit the editor.

Step 9: Test the fstab entry

Run the following command to test the fstab entry:

sudo mount -a

This should mount your Synology NAS share.

Step 10: Reboot and verify

Reboot your Ubuntu system and verify that the Synology NAS share is automatically mounted.

Note: Make sure the /etc/cifs-credentials.txt file has the correct permissions (e.g., chmod 600 /etc/cifs-credentials.txt) and contains the correct username and password.

By adding the entry/etc/fstab, your Ubuntu system will automatically mount the Synology NAS share on startup. That's it! You're all set up and ready to go. Enjoy seamless access to your Synology NAS share!

A straightforward note-taking application saves your text notes in a database and retrieves them when needed. Users can effortlessly create, edit, and remove notes within the app, providing a handy means to organize and monitor crucial details. Featuring a user-friendly interface, the app streamlines the task of storing and accessing vital information.

Requirements:

• Python 12.x

• Tkinter # tkinter —documentation

• Pyinstaller # PyInstaller 6.6.0 documentation

Features:

1. Text note-taking

2. GUI

3. Can be run as script

4. Can be run EXE

5. Editing, deleting options.

6. SQLite database

Installation:

1. Clone the repository: pip git clone https://github.com/emeeran/note_taking_app.git

2. Navigate to the project directory and activate the virtual environment.

3. Pip install tkinter

4. Pip install pyinstaller

pip commands:

Installation

pip install <package name> # install a package.

pip install pywhatkit --force-reinstall # To force reinstall

pip freeze > requirements.txt # To generate a requirements.txt file

pip install -r requirements.txt # To install packages from a requirements.txt file

pip check # To check for conflicting dependencies

pip install more options

pip install comprehensive list

Uninstall

pip uninstall <package name> # uninstall a package

pip uninstall

Update and Upgrade

python -m pip install --upgrade pip # To upgrade pip

py -m pip install --upgrade SomePackage # To upgrade package installed

Checking Installed Packages

pip --version # To know pip version

pip show requests # To check the version of an installed library/module

pip list --outdated # To identify outdated packages

Package Information

pip show --files <package name> # To display the files installed by a package

pip show --verbose <package name> # To show more information about a package

pip show --location <package name> # To show the installation location of a package

pipdeptree # To show a dependency tree of the installed packages

Miscellaneous

pip search <package name> # To search for a package

pip show <package name> # To show information about a package

pip download <package name> # To download a package without installing it

pip wheel <package name> # To create a binary wheel distribution

pip hash <package name> # To compute the hash of a package file

pip config list # To list the current configuration

pip config set <option> <value> # To set a configuration option

pip cache purge # To clear the pip cache

pip freeze # Outputs installed packages in requirements format.

pip check # Verify installed packages have compatible dependencies.

pip search <query> # Search PyPI for packages.

pip wheel <package name> # Build wheels from your requirements.

pip completion # A helper command used for command completion.

pip debug # Show information useful for debugging.

pip cache <command> # Inspect and manage pip's wheel cache.

pip config <command> # Manage local and global configuration.

pip help # Show help for commands.

Dependency Information

pipdeptree # To show a dependency tree of the installed packages

pip completion --bash # To enable bash shell completion

pip completion --zsh # To enable ZSH shell completion

Environment Information

pip debug --verbose # To display detailed information about the pip environment

Troubleshooting

pip check # To check for conflicting dependencies

pip list --outdated # To identify outdated packages

Cache Options

pip cache info # To display information about the cache

pip cache dir # To show the cache directory

pip cache purge # To clear the pip cache

pip cache remove <package name> # To remove a specific package from the cache

pip cache list # To list all packages stored in the cache

Configuration Options

pip config list # To list the current configuration

pip config set <option> <value> # To set a configuration option

pip config unset <option> # To unset a configuration option

pip config edit # To edit the pip configuration file

pip config debug # To display configuration debug information

Debugging and Testing

pip debug # Show information useful for debugging

pip debug --verbose # To display detailed information about the pip environment

pip test <package name> # To test a package

pip test --verbose <package name> # To test a package with verbose output

Download Options

py -m pip download [options] <requirement specifier> [package-index-options] ... # To download with options

py -m pip download [options] -r <requirements file> [package-index-options] ... # To download packages from a requirements file with options

py -m pip download [options] <vcs project url> ... # To download from a version control system (VCS) project URL with options

py -m pip download [options] <local project path> ... # To download from a local project path with options.

py -m pip download [options] <archive url/path> ... # To download from an archived URL or path with options

Shell Completion

pip completion --bash # To enable bash shell completion.

pip completion --zsh # To enable ZSH shell completion.

pip debug --verbose # To display detailed information about the pip environment.

Automating the process of sending WhatsApp messages to multiple people not in your contact list can simplify what is otherwise a tedious task. This project is designed to facilitate communication and save time for those who often send messages to non-contacts on WhatsApp. Simply input the mobile numbers into the contacts.txt file and execute the necessary script to send messages, with the option to schedule them as well.

For these scripts to function, you must have a WhatsApp account and be logged into your browser, as this utilizes WhatsApp Web.

Features:

1. Sending text messages

2. Sending text messages with pictures

3. Sending messages to groups

4. Can schedule messages to be sent at a specific date and time.

Installation:

1. Clone the repository: https://github.com/emeeran/whatsApp_automation_Python.git

2. Navigate to the project directory and activate the virtual environment.

3. pip install pywhatkit

4. pip install pywin32`

Documentation:

pywhatkit: PyPI https://pypi.org/project/pywhatkit (bing.com)

pywin32: pywin32 · PyPI (bing.com)

• git init # initializes.

• git add . # adds all files

• git add <file 1> <filel 2> # adds individual file.

• git commit -m "descriptive message." # commits

• git branch <branch name> # creates a new branch

• git checkout <branch name> # checkouts to a branch

• git branch -n <branch name> # creates new branch and checkouts

• git merge <branch_name> # checkout to the main branch, then merge

• git checkout main/master && git merge <current branch> # merging from being in the branch to be merged.

• git pull origin <branch_name> # pull

• git push origin <branch_name> # push to web repository.

• git branch -d <branch_name> # removing a branch.

• git status # check status.

• ``git remote set-url` # To rename your local Git repository to match the name of the remote repository,

• git rm --cached .env # If the .env file was already tracked by Git before, you’ll need to remove it from the index, and do the following:

  • git add .gitignore git commit -m "Add .env to .gitignore" git push

• git branch # to see local branches.

• git fetch # before listing remote branch, run this.

• git branch -r # to see all remote branches.

• git branch -a # to see both.

• git branch -vv # to see both in detail or.

• git branch -vva # to see both in detail.

• git checkout - # to checkout last branch.

• git branch -m <new name> # to rename current branch

• git branch -m old-branch-name new-branch-name # to rename a branch.

• git push origin -u new-branch-name # push the new branch to remote

Today, I discovered how to transform a Python script into an executable. There are two approaches: one involves using the command line interface (CLI), and the other employs a graphical user interface (GUI). We will now explore CLI method.

Installation:

Python Library: pyinstaller Refer to the documentation for details.

Activate the virtual environment before running pyinstaller.

Step 1: install pyinstaller using pip:

  pip install pyinstaller

Step 2: package your script after installing pyinstaller, navigate to the directory containing your Python script, and run the following command:

pyinstaller --onefile --noconsole your_script.py

This command:

• The --onefile option bundles everything into a single file, simplifying distribution.

• No console window appears when the application launches.

• Ideal for finished GUI applications.

pyinstaller your_script.py

This command:

• Creates a folder with an executable and dependencies.

• Suitable for development and debugging.

After running this command, PyInstaller will create a 'dist' directory in the same location as your script. Within the 'dist' directory, you will find the standalone executable file, which will have the same name as your Python script.

Replace your_script.py with the name of your Python script.

Be aware that PyInstaller includes the Python interpreter, which can increase the size of the executable. Additionally, the executable is specific to the platform it's created on; an executable made on Windows will not work on MacOS or Linux. To support different platforms, you must run PyInstaller on each one individually.

Description.:

The Python project boilerplate is designed to streamline the setup process by automating the creation of essential directories and files, such as 'src' and 'tests', as well as 'README.md' and '.gitignore'. By adhering to best practices, it not only saves developers time but also promotes consistency across projects with standardized structures and formats. Consequently, developers can focus more on coding rather than on the initial project configuration.

Developers can swiftly begin work on their projects without configuring project structures, enabling smooth project initiation. This approach saves time by establishing a clean and organized project structure from the outset, fostering improved project management practices. The tool is designed to boost developer efficiency and productivity by automating project setup, minimizing manual tasks, and encouraging standardized project configurations.

Features:

• creates the basic project structure while adhering to best practices.

• creates a README.md file with a content template.

• Create a .gitignore file with the content required for a Python project.

• Creates a virtual environment.

• Updates PIP and installs Flake8, pre-commits

• executes commands git init, git add ., git commit -m "First Commit".

Installation:

1. Clone the repository:https://github.com/emeeran/python_project_boilerplate.git

2. Install dependencies:pip install -r Requirements.txt.

A Quick Response, otherwise known as a QR code, is a two-dimensional barcode. They are widely popular for their snappy readability and ample storage capacity. Comprised of black squares, you know, arranged on a grid of white squares. It has a lot of capacity for encoding various types of data.

Usage:

This QR code generator is incredibly user-friendly and versatile, catering to a wide audience. Whether you're a small business owner looking to market your products or an event organizer seeking to provide instant access to essential information, this tool is designed to support you. QR codes can significantly enhance communication and enrich the experience for your target audience. Try this tool today and see for yourself how effortlessly you can create and distribute custom QR codes tailored to your unique requirements!

Features:

• QR Code Generation

• UPI Code Generation

• Color Customization

• Size Customization

Installation:

Clone the repository: git clone https://github.com/emeeran/qr_generator.git

Navigate to the project directory: cd qr_generaor

The standard installation uses pypng to make PNG files and can render QR codes straight onto the console, pip install qrcode

For better image functionality, install qrcode along with the pil dependency it will install! pillow and facilitate image generation, pip install "qrcode[pil]"

Library Documentation:

Qrcode: https://pypi.org/project/qrcode/

In this blog post, I'll walk you through the process of creating a Python script to automate project setup. Whether you're a seasoned developer looking to optimize your workflow or a beginner eager to dive into automation, this guide is for you.

Why Automate Project Setup?

Setting up a new project involves repetitive tasks such as creating directories, initializing Git repositories, creating configuration files, and more. These tasks, while necessary, can be time-consuming and error-prone if done manually. By automating the setup process, you can ensure consistency across projects, reduce the chance of human error, and focus more on actual development tasks.

Requirements

For this tutorial, you'll need:

• Basic knowledge of Python programming

• A text editor or an IDE of your choice

• Python installed on your system

Step 1: Define Project Structure

Before diving into coding, it's essential to define the structure of your project. Decide on the directory layout, subdirectories, and files you want to create for each new project. Common subdirectories include src for source code, docs for documentation, data for data files, and .env for environment variables.

Step 2: Write the Python Script

Now, let's start coding! We'll create a Python script that prompts the user for input, such as the parent directory path and project name, and then generates the project structure accordingly.

We'll use Python's built-in os module for file and directory operations and the subprocess module to execute system commands.

Here's the script:

import os
import subprocess


def create_project():
    # Step 1: Prompt for the parent directory path
    parent_dir = input("Enter the parent directory path: ")

    # Step 2: Prompt for project directory name
    project_name = input("Enter the project directory name: ")

    # Step 3: Sub-directories to be hard coded
    subdirectories = [
        "doc",
        "src",
        "data",
        "notes",
        ".env",
    ]

    # Step 4: Files to be created
    files_to_create = [
        "README.md",
        "scribble_pad.md",
    ]

    # Step 5: Create project directory
    project_path = os.path.join(
        parent_dir, project_name.lower()
    )  # Convert to lowercase
    os.makedirs(project_path, exist_ok=True)  # Ensure parent directory exists

    # Step 6: Create subdirectories
    for subdir in subdirectories:
        subdir_path = os.path.join(project_path, subdir)
        os.makedirs(subdir_path, exist_ok=True)  # Ensure subdirectories are created
        # Create main.py and test.py within the "src" subdirectory
        if subdir == "src":
            with open(
                os.path.join(subdir_path, "main.py"), "w"
            ) as f:  # Convert to lowercase
                pass  # Create an empty main.py file
            with open(
                os.path.join(subdir_path, "test.py"), "w"
            ) as f:  # Convert to lowercase
                pass  # Create an empty test.py file

    # Step 7: Create files
    for file_name in files_to_create:
        file_path = os.path.join(
            project_path, file_name.lower()
        )  # Convert to lowercase
        with open(file_path, "w") as f:
            pass  # Create an empty file

    # Step 8: Copy content from root .gitignore to project .gitignore if it exists
    root_gitignore_path = os.path.join(parent_dir, ".gitignore")
    project_gitignore_path = os.path.join(project_path, ".gitignore")
    if os.path.exists(root_gitignore_path):
        with open(root_gitignore_path, "r") as src_file, open(
            project_gitignore_path, "w"
        ) as dest_file:
            dest_file.write(src_file.read())

    # Step 9: Open cmd window and load project directory
    subprocess.run(f"cd {project_path} && start cmd", shell=True)

    # Step 10: Create virtual environment and activate
    subprocess.run(
        f"cd {project_path} && python -m venv .venv && .venv\\Scripts\\activate",
        shell=True,
    )


if __name__ == "__main__":
    create_project()

Step 3: Customize and Expand

Feel free to customize the script according to your specific project requirements. You can add more subdirectories, create additional files, or integrate version control systems like Git.

Step 4: Testing and Refinement

Before using the script in an actual project, it's essential to test it thoroughly in different environments and scenarios.

Step 5: Sharing and Collaboration

Once you're satisfied with your script, consider sharing it with the community! You can publish it on platforms like GitHub, GitLab, or Hashnode, where other developers can discover, use, and contribute to it.

Congratulations! You've learned how to automate project setup using Python scripting. By automating repetitive tasks, you can boost your productivity, maintain consistency across projects, and focus on what matters most—building great software.

Happy coding! 🚀

Modern-day websites can be classified into:

Static websites

These websites display fixed content for all users. The information remains constant and doesn't change unless the webmaster manually edits the HTML source code. URLs often have clear, straightforward structures without parameters. Each page typically corresponds to a specific file or directory.

Dynamic websites

Dynamic websites generate content on the fly, customizing it based on user interactions or other variables. They often use server-side languages, databases, and scripting languages to create a more interactive and personalized experience. URLs may contain parameters or variables, reflecting the dynamic nature of the content. These parameters often influence what content is displayed.

Both dynamic and static mixed websites

Dynamic websites combine elements of both static and dynamic websites. They may have certain pages or sections that are static, while others are generated dynamically. This allows for a balance between personalized content and efficient loading times. These mixed websites often utilize caching techniques to optimize performance and deliver a seamless user experience.

Ways to identify website types:

Static website URLs often have clear, straightforward structures without parameters. Each page typically corresponds to a specific file or directory. The source code is relatively simple and doesn't involve server-side scripting languages like PHP, Python, or Node.js. Limited interactivity, usually basic hyperlinks and forms

Dynamic websites URLs may contain parameters or variables, reflecting the dynamic nature of the content. These parameters often influence what content is displayed. You may find server-side scripting languages in the source code, indicating that content is generated dynamically. Rich interactivity, AJAX requests, and real-time updates are common.

You should be able to tell whether a website is dynamic or static by looking at these indicators. It will take practice and a thorough examination of the HTML code to identify this sort of website.

For scraping both dynamic and static websites, different libraries are often used. Here are some popular choices:

• Static Websites:

  • Beautiful Soup: This Python library is excellent for parsing HTML and XML documents. It's commonly used for extracting data from static web pages.

from bs4 import BeautifulSoup
import requests

url = 'your_static_website_url'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

# Now you can navigate and extract information from 'soup'.

Dynamic Websites:

• • Selenium: When dealing with dynamic content, especially that loaded through JavaScript, Selenium is a powerful tool. It automates browsers, allowing you to interact with the web page as a user would.

from selenium import webdriver
​
url = 'your_dynamic_website_url'
driver = webdriver.Chrome()  # You need to have the appropriate WebDriver installed
driver.get(url)
​
# Now you can interact with the dynamically loaded content using Selenium.
​

• Scrapy with Splash: Scrapy is a versatile web crawling framework, and when combined with Splash, it can handle dynamic content effectively.

import scrapy
from scrapy_splash import SplashRequest
​
class MySpider(scrapy.Spider):
    # Your spider configuration here...
​
    def start_requests(self):
        url = 'your_dynamic_website_url'
        yield SplashRequest(url, self.parse, args={'wait': 0.5})
​
    def parse(self, response):
        # Parse the dynamically loaded content here.
​

Choose the library based on your specific requirements, and consider factors such as ease of use, community support, and the nature of the website you are scraping.

Let's dive into the actual web scraping project. For demonstration purposes, we are now going to scrape a premium paywalled article from https://indianexpress.com/ using Python and the BeautifulSoup4 library. The reason for choosing this website for a demo is that it is static and simple.

I assume you have little knowledge about web scraping and programming and have installed Python 12.x.

Project Workflow:

1. Create a project directory in Windows Explorer and open it in VS Code.

2. Press Ctrl+Shift+P to open the command palette and create a new virtual environment by selecting the option "Python:create virtual environment." As a convention, it is recommended to name the virtual environment folder ".venv.".

3. Once the virtual environment is created, activate it by running the command.venv/Scripts/activate in the VS Code terminal.

4. Install the required libraries. pip install beautifulSoup4

5. visit https://www.gitignore.io/ and generate a gitignore file for your project.

6. Copy the content of the gitignore file and paste it into a new file named .gitignore in the root directory of your project.

7. initialize git git init

8. Create a file in the root directory named main.py.

9. Copy and paste the entire code below into the main.py file and save it..

10. After successful scraping, save the installed packages and their versions in "Requirements.txt" in the root directory of your project. Run this command in the VS Code terminal: pip freeze > Requirements.txt.

This code will extract the text from the article and save it as an HTML file named "article-header as name.html" in the root directory of your project.

import requests
from bs4 import BeautifulSoup
import re
from html import escape

# URL of the web page
url = "https://indianexpress.com/article/explained/explained-climate/kashmir-ladakh-without-snow-why-implications-9110841/"

# Make a GET request to the URL
response = requests.get(url)

# Check if the request was successful (status code 200)
if response.status_code == 200:
    # Parse the HTML content using BeautifulSoup
    soup = BeautifulSoup(response.text, 'html.parser')

    # Extract text from the header <h1></h1> tag
    section_content = soup.find('div', {'id': 'section', 'class': 'ie_single_story_container', 'data-env': 'production'})
    h1_content = section_content.find('h1').text.strip()

    pcl_full_content = soup.find('div', {'id': 'pcl-full-content', 'class': 'story_details'})

    # Check if sub_header <h2></h2> tags are found before accessing its text attribute
    h2_element = pcl_full_content.find('h2')
    h2_content = h2_element.text.strip() if h2_element else ""

    # Find all <p></p> tags and extract their text content
    p_elements = pcl_full_content.find_all('p')
    p_contents = [p.text.strip() for p in p_elements]

    # Create a valid filename by replacing invalid characters with underscores
    valid_filename = re.sub(r'[\/:*?"<>|]', '_', h1_content.lower()) + '.html'

    # Save as HTML file with the valid filename
    with open(valid_filename, 'w', encoding='utf-8') as file:
        file.write(f"<!DOCTYPE html>\n<html>\n<head>\n\t<title>{escape(h1_content)}</title>\n</head>\n<body>\n")
        file.write(f"\t<h1>{escape(h1_content)}</h1>\n")
        file.write(f"\t<h2>{escape(h2_content)}</h2>\n")
        for p_content in p_contents:
            file.write(f"\t<p>{escape(p_content)}</p>\n")
        file.write("</body>\n</html>")

    print(f"Content saved to {valid_filename}")
else:
    print("Failed to retrieve the web page. Status code:", response.status_code)

Features:
-> Remove distraction
-> Read in fullscreen mode
-> Remove advertisements
-> Save in HTML format
-> Print document
-> Read content using a powerful Text to Speech (TTS) engine
-> Edit HTML content (live editor)
-> Email document (with title, body, and reference to the original document)
-> Correctly display mathematical formulas (MathJax equations are Supported)
-> Highlight selected text
-> Move to the next and previous pages
-> Keyboard shortcuts for almost all actions
-> Resize images on the design-mode
-> Display DOI (Digital Object Identifier) details
-> Display publish date

For more info please visit:
http://add0n.com/chrome-reader-view.html

In the fast-paced digital age, harnessing data has become crucial to understand patterns, make informed decisions, or simply collect information. Web scraping or web harvesting, is a technique that helps us do just that. This blog post aims to delve into the utility and efficiency of web scraping, leveraging Python and BeautifulSoup4, an excellent duo for the purpose.

Understanding Web Scraping

Web scraping is a method used to extract large amounts of data from websites. This information is collected and then exported into a format that is more useful for the user, be it a.csv file or database.

Ethical and legal considerations are crucial when scraping a website, as not all websites allow it, and the typical workflow involves requesting, receiving, and extracting information.

Introduction to Python and BeautifulSoup4

Python has become the go-to language for developers involved in web scraping due to its simplicity and robust libraries, such as Beautiful Soup 4.

Beautiful Soup 4 is a Python library designed for parsing HTML and XML documents, converting them into parse trees that can be used to extract data easily.

Web Scraping Techniques with Python and BeautifulSoup4

BeautifulSoup4's powerful functionalities allow users to locate HTML elements quickly. It's possible to extract data by drilling down into tags or directly accessing attributes.

However, there's more to it than this: Web scraping has evolved beyond static HTML pages. It's now exceedingly common to encounter dynamic web pages driven by AJAX. To handle these, one can supplement BeautifulSoup4 with libraries like Selenium or Requests-HTML.

Be aware that the script worked fine today may not work next month as web pages are frequently updated. In that case, you will have to modify your code to get it working again.

Best Practices for Effective Web Scraping

First, write down in a note book the workflow:

1. what data you want?

2. carefully inspect the web page html code

3. Identify the tag, class, and div where the data you need is located.

4. write the code in steps, often ensuring that the code works as expected.

5. comment on each function as a code block so that others can understand.

Web Scraping Project

1. Create a project directory:mkdir web_scraping

2. load the directory:cd web_scraping

3. virtualize the project:python -m venv .venv

4. activate the virtual environment:.venv/scripts/activate

   pip install modules:

    pip install requests BeautifulSoup4
   

import libraries:

import requests
from bs4 import BeautifulSoup
import os
import re

Here is the complete code with comments for easy understanding of each piece of code and its function. It scrapes the text content of a page from https://www.marrowmatters.com/Aplastic-Anemia.html and saves it as a markdown file.

# URL of the website
url = "https://www.marrowmatters.com/Aplastic-Anemia.html"

# Send a GET request to the URL
response = requests.get(url)

# Check if the request was successful (status code 200)
if response.status_code == 200:
    # Parse the HTML content using BeautifulSoup
    soup = BeautifulSoup(response.content, 'html.parser')

    # Extract text within <h1>, <h2>, <h3>, and <p> tags
    h1_texts = [h1.text.strip() for h1 in soup.find_all('h1')]
    h2_and_p_texts = []
    h3_and_p_texts = []

    # Iterate through each <h2> tag
    for h2 in soup.find_all('h2'):
        h2_text = h2.text.strip()
        p_texts = []

        # Find all <p> tags following the current <h2> tag
        next_sibling = h2.find_next_sibling()
        while next_sibling and (next_sibling.name == 'p' or next_sibling.name == 'h3'):
            if next_sibling.name == 'p':
                p_texts.append(next_sibling.text.strip())
            elif next_sibling.name == 'h3':
                h3_text = next_sibling.text.strip()
                h3_p_texts = []

                # Find all <p> tags following the current <h3> tag
                h3_next_sibling = next_sibling.find_next_sibling()
                while h3_next_sibling and h3_next_sibling.name == 'p':
                    h3_p_texts.append(h3_next_sibling.text.strip())
                    h3_next_sibling = h3_next_sibling.find_next_sibling()

                h3_and_p_texts.append((h3_text, h3_p_texts))

            next_sibling = next_sibling.find_next_sibling()

        h2_and_p_texts.append((h2_text, p_texts))

    # Determine the filename based on the first <h1> content
    filename = h1_texts[0] if h1_texts else "extracted_content"

    # Replace invalid characters in the filename
    filename = re.sub(r'[\\/:*?"<>|]', '_', filename)

    # Create the 'data_extracted' directory if it doesn't exist
    output_directory = "data_extracted"
    os.makedirs(output_directory, exist_ok=True)

    # Save the extracted content as a Markdown file
    output_path = os.path.join(output_directory, f"{filename}.md")
    with open(output_path, "w", encoding="utf-8") as file:
        file.write("# " + "\n# ".join(h1_texts) + "\n\n")

        for h2_text, p_texts in h2_and_p_texts:
            file.write(f"## {h2_text}\n")
            file.write("\n".join(f"{text}\n" for text in p_texts))

        for h3_text, h3_p_texts in h3_and_p_texts:
            file.write(f"### {h3_text}\n")
            file.write("\n".join(f"{text}\n" for text in h3_p_texts))

    print(f"Extraction and saving completed successfully. File saved as {filename}.md in {output_directory}")
else:
    print(f"Failed to retrieve the webpage. Status code: {response.status_code}")

Conclusion

Web scraping with Python and BeautifulSoup4 is crucial for efficiently accessing needed information from unstructured data. Enjoy the power of data at your fingertips and be sure to use it responsibly.

Your web scraping journey now awaits. Remember, make sure to respect your data sources, understand what you need, and keep refining your code. With time and practice, you will become proficient at effectively harvesting data from the web. Remember, good web scraping isn’t fulfilled by having data but by having the right data. Happy web scraping!

• Win+R in the run box wt.exe to run terminal

• Use cd command to navigate to the directory where you want your project directory to be placed

• type mkdir your_project_name to create project folder

• type python -m venv .venv to create virtual environment using “venv” module or

• type conda create -n your_project_name python=3.x to create a new environment using "conda" your_project_name.

• type .venv\\Scripts\\activate on Windows or source .env/bin/activate on macOS/Linux. or conda activate your_project_name

• type type nul> your_project_name.py to create a new file for writing script

• type git init to initialize a Git repository within your project directory.

• type git add main.py or git add *

• type git commit -m "Initial commit" to commit the staged changes to the Git repository with a descriptive message.

• Open the VS Code terminal or navigate back to the project directory in your terminal

• type in vs code terminalpip install -r requirements.txt to install any required packages listed in a requirements.txt file within your project.

• type git push origin main to push your local changes to the remote repository

• type git pull origin main to pull any changes from the remote repository

• after finishing work in your project, install pyinstaller pip install pyinstalller

• type pip freeze > requirements.txt to create requirements file

• type pyinstaller your_script.py to convert your script as exe if you want

• type deactivate to deactivate the virtual environment.