Thanks Julien :)

[apps/madmutt.git] / nntp.c

/*
 * Copyright notice from original mutt:
 * Copyright (C) 1998 Brandon Long <blong@fiction.net>
 * Copyright (C) 1999 Andrej Gritsenko <andrej@lucky.net>
 * Copyright (C) 2000-2002 Vsevolod Volkov <vvv@mutt.org.ua>
 *
 * This file is part of mutt-ng, see http://www.muttng.org/.
 * It's licensed under the GNU General Public License,
 * please see the file GPL in the top level source directory.
 */

#include <lib-lib/lib-lib.h>

#include <libgen.h>

#include <lib-mime/mime.h>
#include <lib-ui/lib-ui.h>
#include <lib-ui/sidebar.h>
#include <lib-mx/mx.h>

#include "mutt.h"
#include "sort.h"
#include "nntp.h"
#include "buffy.h"
#include "crypt.h"

#define NNTP_PORT      119
#define NNTP_SSL_PORT  563

static struct {
    unsigned use_cache : 1;

    int  checked;
} nntp = { true, 0 };

static int nntp_check_newgroups (nntp_server_t *, int);

static void nntp_free_acache(nntp_data_t * data)
{
    for (int i = 0; i < countof(data->acache); i++) {
        if (data->acache[i].path) {
            unlink(data->acache[i].path);
            p_delete(&data->acache[i].path);
        }
    }
}

void nntp_data_wipe(nntp_data_t *data)
{
    p_delete(&data->entries);
    p_delete(&data->desc);
    p_delete(&data->cache);
    p_delete(&data->group);
    nntp_free_acache(data);
}

/* newsrc {{{ */

static void mutt_newsgroup_stat(nntp_data_t *data)
{
    data->unread = 0;
    if (data->last == 0 || data->first > data->last)
        return;

    data->unread = data->last - data->first + 1;
    for (int i = 0; i < data->num; i++) {
        int first = MAX(data->entries[i].first, data->first);
        int last  = MIN(data->entries[i].last,  data->last);
        data->unread -= MAX(0, last - first + 1);
    }
}

static int nntp_parse_newsrc_line(nntp_server_t *news, const char *line)
{
    nntp_data_t *data;
    char group[LONG_STRING];
    const char *p;
    int x = 1;

    for (p = line; *p; p++) {
        x += *p == ',';
    }

    p = strpbrk(line, ":!");
    if (!p)
        return -1;

    m_strncpy(group, ssizeof(group), line, p - line);
    data = hash_find(news->newsgroups, group);
    if (!data) {
        data = nntp_data_new();
        data->group = p_dupstr(line, p - line);
        data->nserv = news;
        data->deleted = 1;
        hash_insert(news->newsgroups, data->group, data);
        news->tail = nntp_data_list_append(news->tail, data);
    } else {
        p_delete(&data->entries);
    }

    data->rc = 1;
    data->entries = p_new(NEWSRC_ENTRY, x * 2);
    data->max = x * 2;
    data->subscribed = (*p++ == ':');
    p = skipspaces(p);

    for (x = 0; *p; p++) {
        data->entries[x].first = strtol(p, (char **)&p, 10);
        p += strcspn(p, "-,");
        if (*p == '-') {
            if (!*p)
                break;
            data->entries[x].last = strtol(p + 1, (char **)&p, 10);
        } else {
            data->entries[x].last = data->entries[x].first;
            p = strchrnul(p, ',');
        }
        x += data->entries[x].last != 0;
    }

    if (x && !data->last)
        data->last = data->entries[x - 1].last;
    data->num = x;
    mutt_newsgroup_stat(data);
    return 0;
}

static int slurp_newsrc (nntp_server_t * news)
{
    FILE *fp;
    char *buf = NULL;
    size_t n  = 0;
    struct stat sb;

    news->stat  = stat(news->newsrc, &sb);
    news->size  = sb.st_size;
    news->mtime = sb.st_mtime;

    if ((fp = safe_fopen(news->newsrc, "r")) == NULL)
        return -1;

    /* hmm, should we use dotlock? */
    if (mx_lock_file(news->newsrc, fileno (fp), 0, 0, 1)) {
        m_fclose(&fp);
        return -1;
    }

    while (getline(&buf, &n, fp) >= 0) {
        nntp_parse_newsrc_line(news, buf);
    }
    p_delete(&buf);

    mx_unlock_file (news->newsrc, fileno (fp), 0);
    m_fclose(&fp);
    return 0;
}

#define nntp_cache_expand(dst, dlen, fmt, ...)     \
    do {                                                                     \
        snprintf((dst), (dlen), "%s/" fmt, mod_core.cachedir, ##__VA_ARGS__);\
        mutt_expand_path((dst), (dlen));                                     \
    } while (0)

/* Loads $news_cache_dir/.index into memory, loads newsserver data
 * and newsgroup cache names */
static int nntp_parse_cacheindex(nntp_server_t *news)
{
    char buf[HUGE_STRING];
    FILE *idx;

    p_delete(&news->cache);
    if (m_strisempty(mod_core.cachedir))
        return 0;

    nntp_cache_expand(buf, sizeof(buf), "%s.index", news->conn->account.host);
    if (!(idx = safe_fopen(buf, "a+")))
        return 0;
    rewind(idx);

    while (fgets(buf, sizeof(buf), idx)) {
        buf[m_strlen(buf) - 1] = 0;  /* strip ending '\n' */
        if (!m_strncmp(buf, "#: ", 3)
        &&  !m_strcasecmp(buf + 3, news->conn->account.host)) {
            break;
        }
    }

    while (fgets(buf, sizeof(buf), idx)) {
        char *p, *q;
        int l, m, t;

        if (m_strstart(buf, "#:", NULL))
            break;

        p = strchrnul(buf, ' ');
        if (!*p)
            continue;
        *p++ = '\0';
        p = vskipspaces(p);
        q = strchrnul(p, ' ');
        if (!*q)
            continue;
        *q++ = '\0';
        l = strtol(q, &q, 10);
        m = strtol(q, &q, 10);

        if (!m_strcmp(buf, "ALL")) {
            m_strreplace(&news->cache, m_strdup(p));
            news->newgroups_time = m;
            continue;
        }

        if (news->newsgroups) {
            nntp_data_t *data = hash_find(news->newsgroups, buf);

            if (!data) {
                data = nntp_data_new();
                data->group = m_strdup(buf);
                data->nserv = news;
                data->deleted = 1;
                hash_insert (news->newsgroups, data->group, data);
                news->tail = nntp_data_list_append(news->tail, data);
            }
            m_strreplace(&data->cache, p);

            t = !data->first || data->last < m;
            if (!data->first)
                data->first = l;
            if (data->last < m)
                data->last = m;
            data->lastCached = m;
            if (t || !data->unread)
                mutt_newsgroup_stat(data);
        }
    }

    m_fclose(&idx);
    return 0;
}

/* nntp_parse_url: given an NNPT URL, return host, port,
 * username, password and newsgroup will recognise. */
static int nntp_parse_url(const char *server, ACCOUNT *act, char *group,
                          ssize_t group_len)
{
    ciss_url_t url;
    char s[STRING];

    act->flags = 0;
    act->port  = 0;
    act->type  = M_ACCT_TYPE_NNTP;

    m_strcpy(s, sizeof(s), server);
    url_parse_ciss(&url, s);

    if (url.scheme != U_NNTP && url.scheme != U_NNTPS)
        return -1;

    act->has_ssl = (url.scheme == U_NNTPS);
    if (!act->port) {
        act->port = act->has_ssl ? NNTP_SSL_PORT : NNTP_PORT;
    }
    m_strcpy(group, group_len, url.path);

    return mutt_account_fromurl(act, &url);
}

void nntp_expand_path(char *line, ssize_t len, ACCOUNT *act)
{
    ciss_url_t url = { .path = alloca(len) };

    m_strcpy(url.path, len, line);
    mutt_account_tourl (act, &url);
    url_ciss_tostring(&url, line, len, 0);
}

static int add_group(char *buf, void *serv)
{
    char mod, group[HUGE_STRING], desc[HUGE_STRING];
    nntp_server_t *srv = serv;
    nntp_data_t *data;
    static int n = 0;
    int first, last;

    nntp.checked = n;       /* nntp.checked have N, where N = number of groups */
    if (!buf)               /* at EOF must be zerouth */
        n = 0;

    if (!srv || !buf)
        return 0;

    *desc = '\0';
    sscanf(buf, "%s %d %d %c %[^\n]", group, &last, &first, &mod, desc);

    if (!(data = hash_find(srv->newsgroups, group))) {
        n++;
        data = nntp_data_new();
        data->group = m_strdup(group);
        data->nserv = srv;
        hash_insert(srv->newsgroups, data->group, data);
        srv->tail = nntp_data_list_append(srv->tail, data);
    }

    data->deleted = 0;
    data->first = first;
    data->last  = last;
    data->allowed = mod == 'y';
    m_strreplace(&data->desc, desc);
    mutt_newsgroup_stat(data);
    return 0;
}

static int nntp_get_cache_all(nntp_server_t * serv)
{
    char buf[HUGE_STRING];
    FILE *f;

    nntp_cache_expand(buf, ssizeof(buf), "%s", serv->cache);
    if ((f = fopen(buf, "r"))) {
        int i = 0;

        while (fgets(buf, sizeof(buf), f)) {
            if (ReadInc && (i % ReadInc == 0))
                mutt_message (_("Loading list from cache... %d"), i);
            add_group(buf, serv);
            i++;
        }
        add_group (NULL, NULL);
        m_fclose(&f);
        mutt_clear_error();
        return 0;
    }

    p_delete(&serv->cache);
    return -1;
}

/*
 * Automatically loads a newsrc into memory, if necessary.
 * Checks the size/mtime of a newsrc file, if it doesn't match, load
 * again.  Hmm, if a system has broken mtimes, this might mean the file
 * is reloaded every time, which we'd have to fix.
 *
 * a newsrc file is a line per newsgroup, with the newsgroup, then a 
 * ':' denoting subscribed or '!' denoting unsubscribed, then a 
 * comma separated list of article numbers and ranges.
 */
nntp_server_t *mutt_select_newsserver (char *server)
{
  char file[_POSIX_PATH_MAX];
  char *buf, *p;
  nntp_data_t *list;
  ACCOUNT act;
  nntp_server_t *serv;
  CONNECTION *conn;

  p_clear(&act, 1);

  if (m_strisempty(server)) {
    mutt_error _("No newsserver defined!");
    return NULL;
  }

  buf = p = p_new(char, m_strlen(server) + 10);
  if (url_check_scheme (server) == U_UNKNOWN) {
    strcpy (buf, "nntp://");
    p = strchr (buf, '\0');
  }
  strcpy (p, server);

  if ((nntp_parse_url(buf, &act, file, sizeof(file))) < 0 || *file) {
    p_delete(&buf);
    mutt_error (_("%s is an invalid newsserver specification!"), server);
    return NULL;
  }
  p_delete(&buf);

  conn = mutt_conn_find (NULL, &act);
  if (!conn)
    return NULL;

  nntp_cache_expand(file, sizeof(file), "%s.newsrc", conn->account.host);
  serv = (nntp_server_t *) conn->data;
  if (serv) {
    struct stat sb;

    /* externally modified? */
    if (serv->stat != stat(file, &sb)
    || (!serv->stat && (serv->size != sb.st_size
                        || serv->mtime != sb.st_mtime)))
    {
      for (list = serv->list; list; list = list->next) {
        list->subscribed = list->rc = list->num = 0;
      }
      slurp_newsrc(serv);
    }

    if (serv->status == NNTP_BYE)
      serv->status = NNTP_NONE;
    nntp_check_newgroups(serv, 0);
    return serv;
  }

  /* New newsserver */
  serv = p_new(nntp_server_t, 1);
  serv->tail = &serv->list;
  serv->conn = conn;
  serv->newsrc = m_strdup(file);
  serv->newsgroups = hash_new(SHRT_MAX, false);
  slurp_newsrc(serv);          /* load .newsrc */
  nntp_parse_cacheindex(serv); /* load .index */
  if (nntp.use_cache && serv->cache && nntp_get_cache_all(serv) >= 0) {
    nntp_check_newgroups (serv, 1);
  } else if (nntp_get_active(serv) < 0) {
    hash_delete(&serv->newsgroups, NULL);
    nntp_data_list_wipe(&serv->list);
    p_delete(&serv->newsrc);
    p_delete(&serv->cache);
    p_delete(&serv);
    return NULL;
  }

  conn->data = serv;
  return serv;
}

/* 
 * full status flags are not supported by nntp, but we can fake some
 * of them.  This is how:
 * Read = a read message number is in the .newsrc
 * New = a message is new since we last read this newsgroup
 * Old = anything else
 * So, Read is marked as such in the newsrc, old is anything that is 
 * "skipped" in the newsrc, and new is anything not in the newsrc nor
 * in the cache. By skipped, I mean before the last unread message
 */
static void
nntp_get_status(CONTEXT *ctx, HEADER *h, const char *group, int article)
{
    nntp_data_t *data = ctx->data;

    if (group)
        data = hash_find(data->nserv->newsgroups, group);
    if (!data)
        return;

    for (int i = 0; i < data->num; i++) {
        if ((article >= data->entries[i].first) &&
            (article <= data->entries[i].last))
        {
            /* we cannot use mutt_set_flag() because mx_update_context()
               didn't called yet */
            h->read = 1;
            return;
        }
    }

    /* If article was not cached yet, it is new! :) */
    if (!data->cache || article > data->lastCached)
        return;

    /* Old articles are articles which aren't read but an article after them
     * has been cached */
    if (option(OPTMARKOLD))
        h->old = 1;
}

static void newsrc_gen_entries (CONTEXT * ctx)
{
  nntp_data_t *data = (nntp_data_t *) ctx->data;
  int series, x;
  int last = 0, first = 1;
  int save_sort = SORT_ORDER;

  if (Sort != SORT_ORDER) {
    save_sort = Sort;
    Sort = SORT_ORDER;
    mutt_sort_headers (ctx, 0);
  }

  if (!data->max) {
    data->entries = p_new(NEWSRC_ENTRY, 5);
    data->max = 5;
  }

  /*
   * Set up to fake initial sequence from 1 to the article before the 
   * first article in our list
   */
  data->num = 0;
  series = 1;

  for (x = 0; x < ctx->msgcount; x++) {
    if (series) {               /* search for first unread */
      /*
       * We don't actually check sequential order, since we mark 
       * "missing" entries as read/deleted
       */
      last = ctx->hdrs[x]->article_num;
      if (last >= data->first && !ctx->hdrs[x]->deleted &&
          !ctx->hdrs[x]->read) {
        if (data->num >= data->max) {
          data->max = data->max * 2;
          p_realloc(&data->entries, data->max);
        }
        data->entries[data->num].first = first;
        data->entries[data->num].last = last - 1;
        data->num++;
        series = 0;
      }
    }
    else {                      /* search for first read */

      if (ctx->hdrs[x]->deleted || ctx->hdrs[x]->read) {
        first = last + 1;
        series = 1;
      }
      last = ctx->hdrs[x]->article_num;
    }
  }
  if (series && first <= data->lastLoaded) {
    if (data->num >= data->max) {
      data->max = data->max * 2;
      p_realloc(&data->entries, data->max);
    }
    data->entries[data->num].first = first;
    data->entries[data->num].last = data->lastLoaded;
    data->num++;
  }

  if (save_sort != Sort) {
    Sort = save_sort;
    mutt_sort_headers (ctx, 0);
  }
}

static int mutt_update_list_file (char *filename, char *section,
                                  const char *key, const char *line)
{
  FILE *ifp;
  FILE *ofp;
  char buf[HUGE_STRING];
  char tmpf[_POSIX_PATH_MAX], lnk[_POSIX_PATH_MAX];
  char *c;
  int ext = 0, done = 0, r = 0, l = 0;

  /* if file not exist, create it */
  if ((ifp = safe_fopen (filename, "a")))
    m_fclose(&ifp);
  if (!(ifp = safe_fopen (filename, "r"))) {
    mutt_error (_("Unable to open %s for reading"), filename);
    return -1;
  }
  if (mx_lock_file (filename, fileno (ifp), 0, 0, 1)) {
    m_fclose(&ifp);
    mutt_error (_("Unable to lock %s"), filename);
    return -1;
  }

  /* use m_tempfile() to get a tempfile in the same
   * directory as filename is so that we can follow symlinks
   * via rename(2); as dirname(2) may modify its argument,
   * temporarily use buf as copy of it
   */
  m_strcpy(buf, sizeof(buf), filename);
  ofp = m_tempfile(tmpf, sizeof(tmpf), dirname(buf), filename);
  if (!ofp) {
    m_fclose(&ifp);
    mutt_error (_("Unable to open %s for writing"), tmpf);
    return -1;
  }

  if (section) {
    while (r != EOF && !done && fgets (buf, sizeof (buf), ifp)) {
      r = fputs (buf, ofp);
      c = buf;
      while (*c && *c != '\n') c++;
      c[0] = 0; /* strip EOL */
      if (!strncmp (buf, "#: ", 3) && !m_strcasecmp(buf+3, section))
        done++;
    }
    if (r != EOF && !done) {
      snprintf (buf, sizeof(buf), "#: %s\n", section);
      r = fputs (buf, ofp);
    }
    done = 0;
  }

  while (r != EOF && fgets (buf, sizeof (buf), ifp)) {
    if (ext) {
      c = buf;
      while (*c && (*c != '\r') && (*c != '\n')) c++;
      c--;
      if (*c != '\\') ext = 0;
    } else if ((section && !strncmp (buf, "#: ", 3))) {
      if (!done && line) {
        fputs (line, ofp);
        fputc ('\n', ofp);
      }
      r = fputs (buf, ofp);
      done++;
      break;
    } else if (key && !strncmp (buf, key, strlen(key)) &&
               (!*key || buf[strlen(key)] == ' ')) {
      c = buf;
      ext = 0;
      while (*c && (*c != '\r') && (*c != '\n')) c++;
      c--;
      if (*c == '\\') ext = 1;
      if (!done && line) {
        r = fputs (line, ofp);
        if (*key)
          r = fputc ('\n', ofp);
        done++;
      }
    } else {
      r = fputs (buf, ofp);
    }
  }

  while (r != EOF && fgets (buf, sizeof (buf), ifp))
    r = fputs (buf, ofp);

  /* If there wasn't a line to replace, put it on the end of the file */
  if (r != EOF && !done && line) {
    fputs (line, ofp);
    r = fputc ('\n', ofp);
  }
  mx_unlock_file (filename, fileno (ifp), 0);
  m_fclose(&ofp);
  m_fclose(&ifp);
  if (r == EOF) {
    unlink (tmpf);
    mutt_error (_("Can't write %s"), tmpf);
    return -1;
  }
  lnk[0] = '\0';
  if ((l = readlink (filename, lnk, sizeof(lnk)-1)) > 0)
    lnk[l] = '\0';
  if (rename (tmpf, l > 0 ? lnk : filename) < 0) {
    unlink (tmpf);
    mutt_error (_("Can't rename %s to %s"), tmpf, l > 0 ? lnk : filename);
    return -1;
  }
  return 0;
}

int mutt_newsrc_update (nntp_server_t * news)
{
    buffer_t buf;
    nntp_data_t *data;
    int r = -1;

    if (!news)
        return -1;

    buffer_init(&buf);

    /* we will generate full newsrc here */
    for (data = news->list; data; data = data->next) {
        if (!data || !data->rc)
            continue;

        buffer_addstr(&buf, data->group);
        buffer_addch(&buf, data->subscribed ? ':' : '!');

        for (int x = 0; x < data->num; x++) {
            buffer_addch(&buf, x ? ',' : ' ');
            buffer_addf(&buf, "%d-%d", data->entries[x].first,
                        data->entries[x].last);
        }
        buffer_addch(&buf, '\n');
    }

    /* newrc being fully rewritten */
    if (news->newsrc
    &&  (r = mutt_update_list_file(news->newsrc, NULL, "", buf.data)) == 0)
    {
        struct stat st;

        stat (news->newsrc, &st);
        news->size = st.st_size;
        news->mtime = st.st_mtime;
    }

    buffer_wipe(&buf);
    return r;
}

static FILE *mutt_mkname (char *s)
{
  char buf[_POSIX_PATH_MAX], *pc;
  int fd;
  FILE *fp;

  nntp_cache_expand(buf, ssizeof(buf), "%s", s);
  if ((fp = safe_fopen (buf, "w")))
    return fp;

  nntp_cache_expand(buf, ssizeof(buf), "cache-XXXXXX");
  pc = buf + m_strlen(buf) - 12; /* positioning to "cache-XXXXXX" */
  if ((fd = mkstemp (buf)) == -1)
    return NULL;
  strcpy (s, pc);               /* generated name */
  return fdopen (fd, "w");
}

/* Updates info into .index file: ALL or about selected newsgroup */
static int nntp_update_cacheindex (nntp_server_t * serv, nntp_data_t * data)
{
  char buf[LONG_STRING];
  char file[_POSIX_PATH_MAX];
  const char *key = "ALL";

  if (!serv || !serv->conn || !serv->conn->account.host)
    return -1;

  if (data && data->group) {
    key = data->group;
    snprintf(buf, sizeof(buf), "%s %s %d %d", key, data->cache,
             data->first, data->lastLoaded);
  } else {
    m_strcpy(file, sizeof(file), serv->cache);
    snprintf(buf, sizeof (buf), "ALL %s 0 %d", file,
             (int)serv->newgroups_time);
  }
  nntp_cache_expand(file, ssizeof(file), "%s.index", serv->conn->account.host);
  return mutt_update_list_file(file, serv->conn->account.host, key, buf);
}

static void nntp_delete_cache (nntp_data_t * data)
{
  char buf[_POSIX_PATH_MAX];

  if (!nntp.use_cache || !data || !data->cache || !data->nserv)
    return;

  nntp_cache_expand(buf, ssizeof(buf), "%s", data->cache);
  unlink (buf);
  p_delete(&data->cache);
  data->lastCached = 0;
  nntp_cache_expand(buf, ssizeof(buf), "%s.index",
                    data->nserv->conn->account.host);
  mutt_update_list_file(buf, data->nserv->conn->account.host, data->group,
                        NULL);
}

static int nntp_save_cache_index (nntp_server_t * news)
{
  char buf[HUGE_STRING];
  char file[_POSIX_PATH_MAX];
  nntp_data_t *d;
  FILE *f;

  if (!news || !news->newsgroups)
    return -1;
  if (!nntp.use_cache)
    return 0;

  if (news->cache) {
    nntp_cache_expand(file, ssizeof(file), "%s", news->cache);
    unlink(file);
    f = safe_fopen(file, "w");
  } else {
    m_strcpy(buf, ssizeof(buf), news->conn->account.host);
    f = mutt_mkname(buf);
    news->cache = m_strdup(buf);
    nntp_cache_expand(file, ssizeof(file), "%s", buf);
  }
  if (!f)
    return -1;

  for (d = news->list; d; d = d->next) {
    if (!d->deleted) {
      if (d->desc)
        snprintf (buf, sizeof (buf), "%s %d %d %c %s\n", d->group,
                  d->last, d->first, d->allowed ? 'y' : 'n',
                  d->desc);
      else
        snprintf (buf, sizeof (buf), "%s %d %d %c\n", d->group,
                  d->last, d->first, d->allowed ? 'y' : 'n');
      if (fputs (buf, f) == EOF) {
        m_fclose(&f);
        unlink (file);
        return -1;
      }
    }
  }
  m_fclose(&f);

  if (nntp_update_cacheindex (news, NULL)) {
    unlink (file);
    return -1;
  }
  return 0;
}

static int nntp_save_cache_group (CONTEXT * ctx)
{
  char buf[HUGE_STRING], addr[STRING];
  char file[_POSIX_PATH_MAX];
  FILE *f;
  HEADER *h;
  int i = 0, save = SORT_ORDER;
  int prev = 0;

  if (!nntp.use_cache)
    return 0;
  if (!ctx || !ctx->data || ctx->magic != M_NNTP)
    return -1;

  if (((nntp_data_t *) ctx->data)->cache) {
    nntp_cache_expand(file, ssizeof(file), "%s",
                      ((nntp_data_t *)ctx->data)->cache);
    unlink (file);
    f = safe_fopen (file, "w");
  }
  else {
    snprintf(buf, ssizeof(buf), "%s-%s",
             ((nntp_data_t *)ctx->data)->nserv->conn->account.host,
             ((nntp_data_t *)ctx->data)->group);
    f = mutt_mkname (buf);
    ((nntp_data_t *)ctx->data)->cache = m_strdup(buf);
    nntp_cache_expand(file, ssizeof(file), "%s", buf);
  }
  if (!f)
    return -1;

  if (Sort != SORT_ORDER) {
    save = Sort;
    Sort = SORT_ORDER;
    mutt_sort_headers (ctx, 0);
  }

  /* Save only $nntp_context messages... */
  ((nntp_data_t *) ctx->data)->lastCached = 0;
  if (NntpContext && ctx->msgcount > NntpContext)
    i = ctx->msgcount - NntpContext;
  for (; i < ctx->msgcount; i++) {
    if (!ctx->hdrs[i]->deleted && ctx->hdrs[i]->article_num != prev) {
      char *loc;
      h = ctx->hdrs[i];
      addr[0] = 0;
      rfc822_addrcat(addr, sizeof(addr), h->env->from, 0);

      loc = setlocale(LC_TIME, "C");
      strftime(buf, sizeof(buf), "%d %b %Y %H:%M:%S GMT", gmtime(&h->date_sent));
      setlocale(LC_TIME, loc);

      fprintf(f, "%d\t%s\t%s\t%s\t%s\t\n", h->article_num, h->env->subject,
              addr, buf, h->env->message_id);
      if (h->env->references)
        mutt_write_references (h->env->references, f);
      if (fprintf(f, "\t%zd\t%d\tXref: %s\n", h->content->length, h->lines,
                  NONULL(h->env->xref)) == EOF)
      {
        m_fclose(&f);
        unlink (file);
        return -1;
      }
    }
    prev = ctx->hdrs[i]->article_num;
  }

  if (save != Sort) {
    Sort = save;
    mutt_sort_headers (ctx, 0);
  }
  m_fclose(&f);

  if (nntp_update_cacheindex (((nntp_data_t *) ctx->data)->nserv,
                              (nntp_data_t *) ctx->data)) {
    unlink (file);
    return -1;
  }
  ((nntp_data_t *) ctx->data)->lastCached =
    ((nntp_data_t *) ctx->data)->lastLoaded;
  return 0;
}

nntp_data_t *mutt_newsgroup_subscribe (nntp_server_t * news, char *group)
{
  nntp_data_t *data;

  if (!news || !news->newsgroups || !group || !*group)
    return NULL;
  if (!(data = (nntp_data_t *) hash_find (news->newsgroups, group))) {
    data = nntp_data_new();
    data->group = m_strdup(group);
    data->nserv = news;
    data->deleted = 1;
    hash_insert (news->newsgroups, data->group, data);
    news->tail = nntp_data_list_append(news->tail, data);
  }
  if (!data->subscribed) {
    data->subscribed = 1;
    data->rc = 1;
  }
  return data;
}

nntp_data_t *mutt_newsgroup_unsubscribe (nntp_server_t * news, char *group)
{
  nntp_data_t *data;

  if (!news || !news->newsgroups || !group || !*group ||
      !(data = (nntp_data_t *) hash_find (news->newsgroups, group)))
    return NULL;
  if (data->subscribed) {
    data->subscribed = 0;
  }
  return data;
}

nntp_data_t *mutt_newsgroup_catchup (nntp_server_t * news, char *group)
{
  nntp_data_t *data;

  if (!news || !news->newsgroups || !group || !*group ||
      !(data = (nntp_data_t *) hash_find (news->newsgroups, group)))
    return NULL;
  if (!data->max) {
    data->entries = p_new(NEWSRC_ENTRY, 5);
    data->max = 5;
  }
  data->num = 1;
  data->entries[0].first = 1;
  data->unread = 0;
  data->entries[0].last = data->last;
  if (Context && Context->data == data) {
    int x;

    for (x = 0; x < Context->msgcount; x++)
      mutt_set_flag (Context, Context->hdrs[x], M_READ, 1);
  }
  return data;
}

nntp_data_t *mutt_newsgroup_uncatchup (nntp_server_t * news, char *group)
{
  nntp_data_t *data;

  if (!news || !news->newsgroups || !group || !*group ||
      !(data = (nntp_data_t *) hash_find (news->newsgroups, group)))
    return NULL;
  if (!data->max) {
    data->entries = p_new(NEWSRC_ENTRY, 5);
    data->max = 5;
  }
  data->num = 1;
  data->entries[0].first = 1;
  data->entries[0].last = data->first - 1;
  if (Context && Context->data == data) {
    int x;

    data->unread = Context->msgcount;
    for (x = 0; x < Context->msgcount; x++)
      mutt_set_flag (Context, Context->hdrs[x], M_READ, 0);
  }
  else
    data->unread = data->last - data->entries[0].last;
  return data;
}

/* this routine gives the first newsgroup with new messages */
void nntp_buffy (char* dst, ssize_t dstlen) {
  nntp_data_t *list;
  int count = 0;

  /* forward to current group */
  for (list = CurrentNewsSrv->list; list; list = list->next) {
    if (list->subscribed && list->unread
    &&  Context && Context->magic == M_NNTP
    &&  m_strcmp(list->group, ((nntp_data_t *)Context->data)->group) == 0)
    {
      list = list->next;
      break;
    }
  }

  *dst = '\0';

  while (count < 2) {

    if (!list)
      list = CurrentNewsSrv->list;

    for (; list; list = list->next) {
      if (list->subscribed && list->unread) {
        if (Context && Context->magic == M_NNTP &&
            !m_strcmp(list->group, ((nntp_data_t *)Context->data)->group)) {
          int i, unread = 0;

          for (i = 0; i < Context->msgcount; i++)
            if (!Context->hdrs[i]->read && !Context->hdrs[i]->deleted)
              unread++;
          if (!unread)
            continue;
        }
        m_strcpy(dst, dstlen, list->group);
        break;
      }
    }
    /* done if found */
    if (dst && *dst)
      return;
    count++;
  }
  *dst = '\0';
}

/* }}} */
/* nntp protocol {{{ */

void nntp_sync_sidebar (nntp_data_t* data) {
  int i = 0;
  BUFFY* tmp = NULL;
  char buf[STRING];

  if (!Incoming.len)
    return;

  snprintf(buf, sizeof (buf), "nntp%s://%s%s%s%s/%s",
           data->nserv->conn->account.has_ssl ? "s" : "",
           NONULL(data->nserv->conn->account.user),
           *data->nserv->conn->account.pass ? ":" : "",
           *data->nserv->conn->account.pass ? data->nserv->conn->account.pass : "",
           data->nserv->conn->account.host,
           data->group);

  /* bail out if group not found via mailboxes */
  if ((i = buffy_lookup (buf)) < 0)
    return;

  tmp = Incoming.arr[i];
  /* copied from browser.c */
  if (option (OPTMARKOLD) &&
      data->lastCached >= data->first &&
      data->lastCached <= data->last)
    tmp->msg_unread = data->last - data->lastCached;
  else
    tmp->msg_unread = data->unread;
  tmp->new = data->unread > 0;
  /* this is closest to a "total" count we can get */
  tmp->msgcount = data->last - data->first;
}

static int nntp_auth (nntp_server_t * serv)
{
  CONNECTION *conn = serv->conn;
  char buf[STRING];
  int flags = conn->account.flags;

  if (mutt_account_getuser(&conn->account) || !conn->account.user[0] ||
      mutt_account_getpass(&conn->account) || !conn->account.pass[0]) {
    conn->account.flags = flags;
    return -2;
  }

  mutt_message _("Logging in...");

  snprintf (buf, sizeof (buf), "AUTHINFO USER %s\r\n", conn->account.user);
  mutt_socket_write (conn, buf);
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0) {
    conn->account.flags = flags;
    return -1;
  }

  snprintf (buf, sizeof (buf), "AUTHINFO PASS %s\r\n", conn->account.pass);
  mutt_socket_write(conn, buf);
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0) {
    conn->account.flags = flags;
    return -1;
  }

  if (m_strncmp("281", buf, 3)) {
    conn->account.flags = flags;
    mutt_error _("Login failed.");

    sleep (2);
    return -3;
  }

  return 0;
}

static int nntp_connect_error (nntp_server_t * serv)
{
  serv->status = NNTP_NONE;
  mutt_socket_close (serv->conn);
  mutt_error _("Server closed connection!");

  sleep (2);
  return -1;
}

static int nntp_connect_and_auth (nntp_server_t * serv)
{
  CONNECTION *conn = serv->conn;
  char buf[STRING];
  int rc;

  serv->status = NNTP_NONE;

  if (mutt_socket_open (conn) < 0)
    return -1;

  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return nntp_connect_error (serv);

  if (!m_strncmp("200", buf, 3))
    mutt_message (_("Connected to %s. Posting ok."), conn->account.host);
  else if (!m_strncmp("201", buf, 3))
    mutt_message (_("Connected to %s. Posting NOT ok."), conn->account.host);
  else {
    mutt_socket_close(conn);
    m_strrtrim(buf);
    mutt_error("%s", buf);
    sleep (2);
    return -1;
  }

  /* Tell INN to switch to mode reader if it isn't so. Ignore all
     returned codes and messages. */
  mutt_socket_write (conn, "MODE READER\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return nntp_connect_error (serv);

  mutt_socket_write (conn, "STAT\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return nntp_connect_error (serv);

  if (!conn->account.has_user && m_strncmp("480", buf, 3)) {
    serv->status = NNTP_OK;
    return 0;
  }

  rc = nntp_auth (serv);
  if (rc == -1)
    return nntp_connect_error (serv);
  if (rc == -2) {
    mutt_socket_close (conn);
    serv->status = NNTP_BYE;
    return -1;
  }
  if (rc < 0) {
    mutt_socket_close (conn);
    mutt_error _("Login failed.");

    sleep (2);
    return -1;
  }
  serv->status = NNTP_OK;
  return 0;
}

static int nntp_attempt_features (nntp_server_t * serv)
{
  char buf[LONG_STRING];
  CONNECTION *conn = serv->conn;

  if (serv->feat_known)
      return 0;

  mutt_socket_write (conn, "XOVER\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return nntp_connect_error (serv);
  if (m_strncmp("500", buf, 3))
    serv->hasXOVER = 1;

  mutt_socket_write (conn, "XPAT\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return nntp_connect_error (serv);
  if (m_strncmp("500", buf, 3))
    serv->hasXPAT = 1;

  mutt_socket_write (conn, "LISTGROUP\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return (nntp_connect_error (serv));
  if (m_strncmp("500", buf, 3))
    serv->hasLISTGROUP = 1;

  mutt_socket_write (conn, "XGTITLE +\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
    return nntp_connect_error (serv);
  if (m_strncmp("500", buf, 3))
    serv->hasXGTITLE = 1;

  if (!m_strncmp("282", buf, 3)) {
    do {
      if (mutt_socket_readln (buf, sizeof (buf), conn) < 0)
        return nntp_connect_error (serv);
    } while (!(buf[0] == '.' && buf[1] == '\0'));
  }

  serv->feat_known = 1;
  return 0;
}

static int nntp_open_connection (nntp_server_t * serv)
{
  if (serv->status == NNTP_OK)
    return 0;
  if (serv->status == NNTP_BYE)
    return -1;
  if (nntp_connect_and_auth (serv) < 0)
    return -1;
  if (nntp_attempt_features (serv) < 0)
    return -1;
  return 0;
}

static int nntp_reconnect (nntp_server_t * serv)
{
  char buf[STRING];

  mutt_socket_close (serv->conn);

  for (;;) {
    if (nntp_connect_and_auth (serv) == 0)
      return 0;

    snprintf (buf, sizeof (buf), _("Connection to %s lost. Reconnect?"),
              serv->conn->account.host);
    if (query_quadoption (OPT_NNTPRECONNECT, buf) != M_YES) {
      serv->status = NNTP_BYE;
      return -1;
    }
  }
}

/* Send data from line[LONG_STRING] and receive answer to same line */
static int mutt_nntp_query (nntp_data_t * data, char *line, size_t linelen)
{
  char buf[LONG_STRING];
  int done = TRUE;

  if (data->nserv->status == NNTP_BYE)
    return -1;

  do {
    if (*line) {
      mutt_socket_write (data->nserv->conn, line);
    }
    else if (data->group) {
      snprintf (buf, sizeof (buf), "GROUP %s\r\n", data->group);
      mutt_socket_write (data->nserv->conn, buf);
    }

    done = TRUE;
    if (mutt_socket_readln (buf, sizeof (buf), data->nserv->conn) < 0) {
      if (nntp_reconnect (data->nserv) < 0)
        return -1;

      if (data->group) {
        snprintf (buf, sizeof (buf), "GROUP %s\r\n", data->group);
        mutt_socket_write (data->nserv->conn, buf);
        if (mutt_socket_readln (buf, sizeof (buf), data->nserv->conn) < 0)
          return -1;
      }
      if (*line)
        done = FALSE;
    }
    else if ((!m_strncmp("480", buf, 3)) && nntp_auth (data->nserv) < 0)
      return -1;
  } while (!done);

  m_strcpy(line, linelen, buf);
  return 0;
}

/*
 * This function calls  funct(*line, *data)  for each received line,
 * funct(NULL, *data)  if  rewind(*data)  needs, exits when fail or done.
 * Returned codes:
 *  0 - successful,
 *  1 - correct but not performed (may be, have to be continued),
 * -1 - conection lost,
 * -2 - invalid command or execution error,
 * -3 - error in funct(*line, *data).
 */
static int mutt_nntp_fetch (nntp_data_t * nntp_data, const char *query,
                            const char *msg, progress_t* bar,
                            int (*funct) (char *, void *),
                            void *data, int tagged)
{
  char buf[LONG_STRING];
  char *inbuf, *p;
  int done = FALSE;
  int chunk, line;
  long pos = 0;
  size_t lenbuf = 0;
  int ret;

  do {
    m_strcpy(buf, sizeof(buf), query);
    if (mutt_nntp_query (nntp_data, buf, sizeof (buf)) < 0)
      return -1;
    if (buf[0] == '5')
      return -2;
    if (buf[0] != '2')
      return 1;

    ret = 0;
    line = 0;
    inbuf = p_new(char, sizeof(buf));

    for (;;) {
      chunk = mutt_socket_readln(buf, sizeof (buf), nntp_data->nserv->conn);
      if (chunk < 0)
        break;

      p = buf;
      if (!lenbuf && buf[0] == '.') {
        if (buf[1] == '\0') {
          done = TRUE;
          break;
        }
        if (buf[1] == '.')
          p++;
      }

      m_strcpy(inbuf + lenbuf, sizeof(buf), p);
      pos += chunk;

      if (chunk >= ssizeof (buf)) {
        lenbuf += m_strlen(p);
      }
      else {
        if (bar) {
          mutt_progress_bar (bar, pos);
        } else if (msg) {
          line++;
          if (ReadInc && (line % ReadInc == 0)) {
            if (tagged)
              mutt_message (_("%s (tagged: %d) %d"), msg, tagged, line);
            else
              mutt_message ("%s %d", msg, line);
          }
        }

        if (ret == 0 && funct (inbuf, data) < 0)
          ret = -3;
        lenbuf = 0;
      }

      p_realloc(&inbuf, lenbuf + sizeof (buf));
    }
    p_delete(&inbuf);
    funct (NULL, data);
  }
  while (!done);
  return ret;
}

static int nntp_read_tempfile (char *line, void *file)
{
  FILE *f = (FILE *) file;

  if (!line)
    rewind (f);
  else {
    fputs (line, f);
    if (fputc ('\n', f) == EOF)
      return -1;
  }
  return 0;
}

static void nntp_parse_xref (CONTEXT * ctx, char *group, char *xref,
                             HEADER * h)
{
  register char *p, *b;
  register char *colon = NULL;

  b = p = xref;
  while (*p) {
    /* skip to next word */
    b = p;
    while (*b && ((*b == ' ') || (*b == '\t')))
      b++;
    p = b;
    colon = NULL;
    /* skip to end of word */
    while (*p && (*p != ' ') && (*p != '\t')) {
      if (*p == ':')
        colon = p;
      p++;
    }
    if (*p) {
      *p = '\0';
      p++;
    }
    if (colon) {
      *colon = '\0';
      colon++;
      nntp_get_status (ctx, h, b, atoi (colon));
      if (h && h->article_num == 0 && m_strcmp(group, b) == 0)
        h->article_num = atoi (colon);
    }
  }
}

/*
 * returns:
 *  0 on success
 *  1 if article not found
 * -1 if read or write error on tempfile or socket
 */
static int nntp_read_header (CONTEXT * ctx, const char *msgid,
                             int article_num)
{
  nntp_data_t *nntp_data = ((nntp_data_t *) ctx->data);
  FILE *f;
  char buf[LONG_STRING];
  char tempfile[_POSIX_PATH_MAX];
  int ret;
  HEADER *h = ctx->hdrs[ctx->msgcount];

  f = m_tempfile(tempfile, sizeof(tempfile), NONULL(mod_core.tmpdir), NULL);
  if (!f)
    return -1;

  if (!msgid)
    snprintf (buf, sizeof (buf), "HEAD %d\r\n", article_num);
  else
    snprintf (buf, sizeof (buf), "HEAD %s\r\n", msgid);

  ret = mutt_nntp_fetch (nntp_data, buf, NULL, NULL, nntp_read_tempfile, f, 0);
  if (ret) {
    m_fclose(&f);
    unlink (tempfile);
    return (ret == -1 ? -1 : 1);
  }

  h->article_num = article_num;
  h->env = mutt_read_rfc822_header (f, h, 0, 0);
  m_fclose(&f);
  unlink (tempfile);

  if (h->env->xref != NULL)
    nntp_parse_xref (ctx, nntp_data->group, h->env->xref, h);
  else if (h->article_num == 0 && msgid) {
    snprintf (buf, sizeof (buf), "STAT %s\r\n", msgid);
    if (mutt_nntp_query (nntp_data, buf, sizeof (buf)) == 0)
      h->article_num = atoi (buf + 4);
  }

  return 0;
}

static int parse_description (char *line, void *n)
{
  register char *d = line;
  nntp_data_t *data;
  nntp_server_t *news = n;

  if (!line)
    return 0;
  while (*d && *d != '\t' && *d != ' ')
    d++;
  *d = 0;
  d++;
  while (*d && (*d == '\t' || *d == ' '))
    d++;
  if ((data = (nntp_data_t *) hash_find (news->newsgroups, line)) != NULL &&
      m_strcmp(d, data->desc)) {
    p_delete(&data->desc);
    data->desc = m_strdup(d);
  }
  return 0;
}

static void nntp_get_desc (nntp_data_t * data, const char *mask, char *msg, progress_t* bar)
{
  char buf[STRING];

  if (!option (OPTLOADDESC) || !data || !data->nserv)
    return;

  /* Get newsgroup description, if we can */
  if (data->nserv->hasXGTITLE)
    snprintf (buf, sizeof (buf), "XGTITLE %s\r\n", mask);
  else
    snprintf (buf, sizeof (buf), "LIST NEWSGROUPS %s\r\n", mask);
  mutt_nntp_fetch (data, buf, msg, bar, parse_description, data->nserv, 0);
}

/*
 * XOVER returns a tab separated list of:
 * id|subject|from|date|Msgid|references|bytes|lines|xref
 *
 * This has to duplicate some of the functionality of 
 * mutt_read_rfc822_header(), since it replaces the call to that (albeit with
 * a limited number of headers which are "parsed" by placement in the list)
 */
static int nntp_parse_xover (CONTEXT * ctx, char *buf, HEADER * hdr)
{
  nntp_data_t *nntp_data = (nntp_data_t *) ctx->data;
  char *p, *b;
  int x, done = 0;

  hdr->env = envelope_new();
  hdr->env->newsgroups = m_strdup(nntp_data->group);
  hdr->content = body_new();
  hdr->content->type = TYPETEXT;
  hdr->content->subtype = m_strdup("plain");
  hdr->content->encoding = ENC7BIT;
  hdr->content->disposition = DISPINLINE;
  hdr->content->length = -1;
  b = p = buf;

  for (x = 0; !done && x < 9; x++) {
    /* if from file, need to skip newline character */
    while (*p && *p != '\n' && *p != '\t')
      p++;
    if (!*p)
      done++;
    *p = '\0';
    p++;
    switch (x) {
    case 0:
      hdr->article_num = atoi (b);
      nntp_get_status (ctx, hdr, NULL, hdr->article_num);
      break;
    case 1:
      hdr->env->subject = m_strdup(b);
      break;
    case 2:
      address_list_wipe(&hdr->env->from);
      hdr->env->from = rfc822_parse_adrlist (hdr->env->from, b);
      /* same as for mutt_parse_rfc822_line():
       * don't leave from info NULL if there's an invalid address (or
       * whatever) in From: field; mutt would just display it as empty
       * and mark mail/(esp.) news article as your own. aaargh! this
       * bothered me for _years_ */
      if (!hdr->env->from) {
        hdr->env->from = address_new ();
        hdr->env->from->personal = m_strdup(b);
      }
      break;
    case 3:
      hdr->date_sent = mutt_parse_date (b, hdr);
      hdr->received = hdr->date_sent;
      break;
    case 4:
      p_delete(&hdr->env->message_id);
      hdr->env->message_id = m_strdup(b);
      break;
    case 5:
      string_list_wipe(&hdr->env->references);
      hdr->env->references = mutt_parse_references (b, 0);
      break;
    case 6:
      hdr->content->length = atoi (b);
      break;
    case 7:
      hdr->lines = atoi (b);
      break;
    case 8:
      if (!hdr->read)
        p_delete(&hdr->env->xref);
      b = b + 6;                /* skips the "Xref: " */
      hdr->env->xref = m_strdup(b);
      nntp_parse_xref (ctx, nntp_data->group, b, hdr);
    }
    rfc2047_decode_envelope(hdr->env);
    if (!*p)
      return -1;
    b = p;
  }
  return 0;
}

typedef struct {
  CONTEXT *ctx;
  int   first;
  int   last;
  bits_t messages;
  const char *msg;
} FETCH_CONTEXT;

static int nntp_fetch_numbers (char *line, void *c)
{
    FETCH_CONTEXT *fc = c;

    if (line) {
        int num = atoi(line);
        if (num < fc->first || num > fc->last)
            return 0;

        bit_set(&fc->messages, num);
    }
    return 0;
}

static int add_xover_line (char *line, void *c)
{
  int num, total;
  FETCH_CONTEXT *fc = c;
  CONTEXT *ctx = fc->ctx;
  nntp_data_t *data = (nntp_data_t *) ctx->data;

  if (!line)
    return 0;

  if (ctx->msgcount >= ctx->hdrmax)
    mx_alloc_memory (ctx);
  ctx->hdrs[ctx->msgcount] = header_new();
  ctx->hdrs[ctx->msgcount]->index = ctx->msgcount;

  nntp_parse_xover (ctx, line, ctx->hdrs[ctx->msgcount]);
  num = ctx->hdrs[ctx->msgcount]->article_num;

  if (num >= fc->first && num <= fc->last) {
    ctx->msgcount++;
    if (num > data->lastLoaded)
      data->lastLoaded = num;
    num = num - fc->first + 1;
    total = fc->last - fc->first + 1;
    if (!ctx->quiet && fc->msg && ReadInc && (num % ReadInc == 0))
      mutt_message ("%s %d/%d", fc->msg, num, total);
  } else {
    header_delete(&ctx->hdrs[ctx->msgcount]);       /* skip it */
  }

  return 0;
}


static int nntp_fetch_headers(CONTEXT * ctx, int first, int last)
{
  char buf[HUGE_STRING];
  const char *msg = _("Fetching message headers...");
  const char *msg2 = _("Fetching headers from cache...");
  nntp_data_t *nntp_data = ((nntp_data_t *) ctx->data);
  int ret, num, oldmsgcount, current;
  FILE *f;
  FETCH_CONTEXT fc;

  /* if empty group or nothing to do */
  if (!last || first > last)
    return 0;

  /* fetch list of articles */
  mutt_message _("Fetching list of articles...");

  fc.ctx   = ctx;
  fc.first = first;
  fc.last  = last;

  /* CACHE: must be loaded xover cache here */
  num = nntp_data->lastCached - first + 1;
  if (nntp.use_cache && nntp_data->cache && num > 0) {
  nntp_cache_expand(buf, ssizeof(buf), "%s", nntp_data->cache);
  mutt_message (msg2);

  if ((f = safe_fopen (buf, "r"))) {
    int r = 0, c = 0;

    /* counting number of lines */
    while (fgets (buf, sizeof (buf), f) != NULL)
      r++;
    rewind (f);
    while (r > num && fgets (buf, sizeof (buf), f) != NULL)
      r--;
    oldmsgcount = ctx->msgcount;
    fc.first = first;
    fc.last = first + num - 1;
    fc.msg = NULL;
    while (fgets (buf, sizeof (buf), f) != NULL) {
      if (ReadInc && ((++c) % ReadInc == 0))
        mutt_message ("%s %d/%d", msg2, c, r);
      add_xover_line (buf, &fc);
    }
    m_fclose(&f);
    nntp_data->lastLoaded = fc.last;
    first = fc.last + 1;
    if (ctx->msgcount > oldmsgcount)
      mx_update_context (ctx, ctx->msgcount - oldmsgcount);
  }
  else
    nntp_delete_cache (nntp_data);
  }
  num = last - first + 1;
  if (num <= 0) {
    return 0;
  }

  /*
  * Without XOVER, we have to fetch each article header and parse
  * it.  With XOVER, we ask for all of them
  */
  mutt_message (msg);
  if (nntp_data->nserv->hasXOVER) {
    oldmsgcount = ctx->msgcount;
    fc.first = first;
    fc.last = last;
    fc.msg = msg;
    snprintf (buf, sizeof (buf), "XOVER %d-%d\r\n", first, last);
    ret = mutt_nntp_fetch (nntp_data, buf, NULL, NULL, add_xover_line, &fc, 0);
    if (ctx->msgcount > oldmsgcount)
      mx_update_context (ctx, ctx->msgcount - oldmsgcount);
    if (ret != 0) {
      mutt_error (_("XOVER command failed: %s"), buf);
      return -1;
    }
  } else {
    bits_init(&fc.messages);

    if (nntp_data->nserv->hasLISTGROUP) {
      snprintf (buf, sizeof (buf), "LISTGROUP %s\r\n", nntp_data->group);
      if (mutt_nntp_fetch(nntp_data, buf, NULL, NULL,
                          nntp_fetch_numbers, &fc, 0))
      {
        mutt_error (_("LISTGROUP command failed: %s"), buf);
        sleep (2);
        bits_wipe(&fc.messages);
        return -1;
      }
    } else {
      for (num = first; num <= last; num++)
        bit_set(&fc.messages, num);
    }

    for (current = first; current <= last; current++) {
      HEADER *h;

      ret = current - first + 1;
      mutt_message ("%s %d/%d", msg, ret, num);

      if (!bit_isset(&fc.messages, current))
        continue;

      if (ctx->msgcount >= ctx->hdrmax)
        mx_alloc_memory (ctx);
      h = ctx->hdrs[ctx->msgcount] = header_new();
      h->index = ctx->msgcount;

      ret = nntp_read_header (ctx, NULL, current);
      if (ret == 0) {           /* Got article. Fetch next header */
        nntp_get_status (ctx, h, NULL, h->article_num);
        ctx->msgcount++;
        mx_update_context (ctx, 1);
      }
      else
        header_delete(&h);  /* skip it */
      if (ret == -1) {
        bits_wipe(&fc.messages);
        return -1;
      }

      if (current > nntp_data->lastLoaded)
        nntp_data->lastLoaded = current;
    }
    bits_wipe(&fc.messages);
  }

  nntp_data->lastLoaded = last;
  mutt_clear_error ();
  return 0;
}

/*
 * currently, nntp "mailbox" is "newsgroup"
 */
static int nntp_open_mailbox (CONTEXT * ctx)
{
  nntp_data_t *nntp_data;
  nntp_server_t *serv;
  char buf[HUGE_STRING];
  char server[LONG_STRING];
  int count = 0, first;
  ACCOUNT act;

  p_clear(&act, 1);

  if (nntp_parse_url (ctx->path, &act, buf, sizeof (buf)) < 0 || !*buf) {
    mutt_error (_("%s is an invalid newsgroup specification!"), ctx->path);
    mutt_sleep (2);
    return -1;
  }

  server[0] = '\0';
  nntp_expand_path (server, sizeof (server), &act);
  if (!(serv = mutt_select_newsserver (server)) || serv->status != NNTP_OK)
    return -1;

  CurrentNewsSrv = serv;

  /* create NNTP-specific state struct if nof found in list */
  if ((nntp_data = hash_find(serv->newsgroups, buf)) == NULL) {
    nntp_data = nntp_data_new();
    nntp_data->group = m_strdup(buf);
    hash_insert(serv->newsgroups, nntp_data->group, nntp_data);
    serv->tail = nntp_data_list_append(serv->tail, nntp_data);
  }
  ctx->data = nntp_data;
  nntp_data->nserv = serv;

  mutt_message (_("Selecting %s..."), nntp_data->group);

  if (!nntp_data->desc) {
    nntp_get_desc (nntp_data, nntp_data->group, NULL, NULL);
    if (nntp_data->desc)
      nntp_save_cache_index (serv);
  }

  buf[0] = 0;
  if (mutt_nntp_query (nntp_data, buf, sizeof (buf)) < 0) {
    return -1;
  }

  if (m_strncmp("211", buf, 3)) {
    nntp_data_t **l;

    /* GROUP command failed */
    if (!m_strncmp("411", buf, 3)) {
      mutt_error (_("Newsgroup %s not found on server %s"),
                  nntp_data->group, serv->conn->account.host);

      /* CACHE: delete cache and line from .index */
      nntp_delete_cache (nntp_data);
      hash_remove(serv->newsgroups, nntp_data->group, NULL, NULL);
      for (l = &serv->list; *l; l = &(*l)->next) {
          if ((*l) == nntp_data) {
              nntp_data_list_pop(l);
              nntp_data_delete(&nntp_data);
              if (!*l)
                  serv->tail = l;
              break;
          }
      }
      nntp_data_delete(&nntp_data);
      sleep (2);
    }

    return -1;
  }

  sscanf (buf + 4, "%d %u %u %s", &count, &nntp_data->first,
          &nntp_data->last, buf);

  nntp_data->deleted = 0;

  time (&serv->check_time);

  /*
   * Check for max adding context. If it is greater than $nntp_context,
   * strip off extra articles
   */
  first = nntp_data->first;
  if (NntpContext && nntp_data->last - first + 1 > NntpContext)
    first = nntp_data->last - NntpContext + 1;
  if (first)
    nntp_data->lastLoaded = first - 1;
  return nntp_fetch_headers (ctx, first, nntp_data->last);
}

int nntp_fetch_message (MESSAGE * msg, CONTEXT * ctx, int msgno)
{
  char buf[LONG_STRING];
  char path[_POSIX_PATH_MAX];
  NNTP_CACHE *cache;
  int ret;
  progress_t bar;

  /* see if we already have the message in our cache */
  cache =
    &((nntp_data_t *) ctx->data)->acache[ctx->hdrs[msgno]->index %
                                       NNTP_CACHE_LEN];

  /* if everything is fine, assign msg->fp and return */
  if (cache->path && cache->index == ctx->hdrs[msgno]->index &&
      (msg->fp = fopen (cache->path, "r")))
    return 0;

  /* clear the previous entry */
  unlink (cache->path);
  p_delete(&cache->path);

  cache->index = ctx->hdrs[msgno]->index;
  msg->fp = m_tempfile(path, sizeof(path), NONULL(mod_core.tmpdir), NULL);
  if (!msg->fp) {
    return -1;
  }
  cache->path = m_strdup(path);

  if (ctx->hdrs[msgno]->article_num == 0)
    snprintf (buf, sizeof (buf), "ARTICLE %s\r\n",
              ctx->hdrs[msgno]->env->message_id);
  else
    snprintf (buf, sizeof (buf), "ARTICLE %d\r\n",
              ctx->hdrs[msgno]->article_num);

  bar.msg = _("Fetching message...");
  bar.size = 0;
  mutt_progress_bar (&bar, 0);

  ret = mutt_nntp_fetch ((nntp_data_t *) ctx->data, buf, NULL, &bar, nntp_read_tempfile,
                         msg->fp, ctx->tagged);
  if (ret == 1) {
    mutt_error (_("Article %d not found on server"),
                ctx->hdrs[msgno]->article_num);
  }

  if (ret) {
    m_fclose(&msg->fp);
    unlink (path);
    p_delete(&cache->path);
    return -1;
  }

  envelope_delete(&ctx->hdrs[msgno]->env);
  ctx->hdrs[msgno]->env =
    mutt_read_rfc822_header (msg->fp, ctx->hdrs[msgno], 0, 0);
  /* fix content length */
  fseeko (msg->fp, 0, SEEK_END);
  ctx->hdrs[msgno]->content->length = ftello (msg->fp) -
    ctx->hdrs[msgno]->content->offset;

  /* this is called in mutt before the open which fetches the message, 
   * which is probably wrong, but we just call it again here to handle
   * the problem instead of fixing it.
   */
  mutt_parse_mime_message (ctx, ctx->hdrs[msgno]);

  /* These would normally be updated in mx_update_context(), but the 
   * full headers aren't parsed with XOVER, so the information wasn't
   * available then.
   */
  ctx->hdrs[msgno]->security = crypt_query (ctx->hdrs[msgno]->content);

  mutt_clear_error ();
  rewind (msg->fp);

  return 0;
}

/* Post article */
int nntp_post (const char *msg)
{
  char buf[LONG_STRING];
  size_t len;
  FILE *f;
  nntp_data_t *nntp_data;

  if (Context && Context->magic == M_NNTP)
    nntp_data = (nntp_data_t *)Context->data;
  else {
    if (!(CurrentNewsSrv = mutt_select_newsserver(NewsServer)) ||
        !CurrentNewsSrv->list)
    {
      mutt_error (_("Can't post article. No connection to news server."));
      return -1;
    }
    nntp_data = CurrentNewsSrv->list;
  }

  if (!(f = safe_fopen (msg, "r"))) {
    mutt_error (_("Can't post article. Unable to open %s"), msg);
    return -1;
  }

  m_strcpy(buf, sizeof(buf), "POST\r\n");
  if (mutt_nntp_query (nntp_data, buf, sizeof (buf)) < 0) {
    mutt_error (_("Can't post article. Connection to %s lost."),
                nntp_data->nserv->conn->account.host);
    return -1;
  }
  if (buf[0] != '3') {
    mutt_error (_("Can't post article: %s"), buf);
    return -1;
  }

  buf[0] = '.';
  buf[1] = '\0';
  while (fgets (buf + 1, sizeof (buf) - 2, f) != NULL) {
    len = m_strlen(buf);
    if (buf[len - 1] == '\n') {
      buf[len - 1] = '\r';
      buf[len] = '\n';
      len++;
      buf[len] = '\0';
    }
    if (buf[1] == '.')
      mutt_socket_write(nntp_data->nserv->conn, buf);
    else
      mutt_socket_write(nntp_data->nserv->conn, buf + 1);
  }
  m_fclose(&f);

  if (buf[m_strlen(buf) - 1] != '\n')
    mutt_socket_write(nntp_data->nserv->conn, "\r\n");
  mutt_socket_write(nntp_data->nserv->conn, ".\r\n");
  if (mutt_socket_readln (buf, sizeof (buf), nntp_data->nserv->conn) < 0) {
    mutt_error (_("Can't post article. Connection to %s lost."),
                nntp_data->nserv->conn->account.host);
    return -1;
  }
  if (buf[0] != '2') {
    mutt_error (_("Can't post article: %s"), buf);
    return -1;
  }

  return 0;
}

/* nntp_logout_all: close all open connections. */
void nntp_logout_all (void)
{
  char buf[LONG_STRING];
  CONNECTION *conn;

  conn = mutt_socket_head ();

  while (conn) {
    CONNECTION* next = conn->next;
    if (conn->account.type == M_ACCT_TYPE_NNTP) {
      mutt_message (_("Closing connection to %s..."), conn->account.host);
      mutt_socket_write (conn, "QUIT\r\n");
      mutt_socket_readln (buf, sizeof (buf), conn);
      mutt_clear_error ();
      mutt_socket_close (conn);
      mutt_socket_free (conn);
    }
    conn = next;
  }
}

static int nntp_sync_mailbox (CONTEXT * ctx, int unused1, int* unused2)
{
  nntp_data_t *data = ctx->data;

  nntp_save_cache_group(ctx);
  nntp_free_acache(data);

  data->nserv->check_time = 0;  /* next nntp_check_mailbox() will really check */
  return 0;
}

static void nntp_fastclose_mailbox (CONTEXT * ctx)
{
  nntp_data_t *data = (nntp_data_t *) ctx->data, *tmp;

  if (!data)
    return;
  nntp_free_acache (data);
  if (!data->nserv || !data->nserv->newsgroups || !data->group)
    return;
  nntp_save_cache_index (data->nserv);
  if ((tmp = hash_find (data->nserv->newsgroups, data->group)) == NULL
      || tmp != data)
    nntp_data_delete(&data);
  else
    nntp_sync_sidebar (data);
}

/* commit changes and terminate connection */
int nntp_close_mailbox (CONTEXT * ctx)
{
  if (!ctx)
    return -1;
  mutt_message _("Quitting newsgroup...");

  if (ctx->data) {
    nntp_data_t *data = (nntp_data_t *) ctx->data;
    int ret;

    if (data->nserv && data->nserv->conn && ctx->unread) {
      ret = query_quadoption (OPT_CATCHUP, _("Mark all articles read?"));
      if (ret == M_YES)
        mutt_newsgroup_catchup (data->nserv, data->group);
      else if (ret < 0)
        return -1;
    }
  }
  nntp_sync_mailbox (ctx, 0, NULL);
  if (ctx->data && ((nntp_data_t *) ctx->data)->nserv) {
    nntp_server_t *news;

    news = ((nntp_data_t *) ctx->data)->nserv;
    newsrc_gen_entries (ctx);
    ((nntp_data_t *) ctx->data)->unread = ctx->unread;
    mutt_newsrc_update (news);
  }
  mutt_clear_error ();
  return 0;
}

/* use the GROUP command to poll for new mail */
static int _nntp_check_mailbox (CONTEXT * ctx, nntp_data_t * nntp_data)
{
  char buf[LONG_STRING];
  int count = 0;

  if (nntp_data->nserv->check_time + NewsPollTimeout > time (NULL))
    return 0;

  buf[0] = 0;
  if (mutt_nntp_query (nntp_data, buf, sizeof (buf)) < 0) {
    return -1;
  }
  if (m_strncmp("211", buf, 3)) {
    buf[0] = 0;
    if (mutt_nntp_query (nntp_data, buf, sizeof (buf)) < 0) {
      return -1;
    }
  }
  if (!m_strncmp("211", buf, 3)) {
    int first;
    int last;

    sscanf (buf + 4, "%d %d %d", &count, &first, &last);
    nntp_data->first = first;
    nntp_data->last  = last;
    if (ctx && last > nntp_data->lastLoaded) {
      nntp_fetch_headers (ctx, nntp_data->lastLoaded + 1, last);
      time (&nntp_data->nserv->check_time);
      return 1;
    }
    if (!last || (!nntp_data->rc && !nntp_data->lastCached))
      nntp_data->unread = count;
    else
      mutt_newsgroup_stat (nntp_data);
    /* active was renumbered? */
    if (last < nntp_data->lastLoaded) {
      if (!nntp_data->max) {
        nntp_data->entries = p_new(NEWSRC_ENTRY, 5);
        nntp_data->max = 5;
      }
      nntp_data->lastCached = 0;
      nntp_data->num = 1;
      nntp_data->entries[0].first = 1;
      nntp_data->entries[0].last = 0;
    }
    nntp_sync_sidebar (nntp_data);
  }

  time (&nntp_data->nserv->check_time);
  return 0;
}

static int nntp_check_mailbox (CONTEXT * ctx, int* unused1, int unused2)
{
  return _nntp_check_mailbox (ctx, (nntp_data_t *) ctx->data);
}

static int nntp_check_newgroups (nntp_server_t * serv, int force)
{
  char buf[LONG_STRING];
  nntp_data_t nntp_data;
  nntp_data_t *l, **emp;
  time_t now;
  struct tm *t;

  if (!serv || !serv->newgroups_time)
    return -1;

  if (nntp_open_connection (serv) < 0)
    return -1;

  /* check subscribed groups for new news */
  if (option (OPTSHOWNEWNEWS)) {
    mutt_message _("Checking for new messages...");

    for (l = serv->list; l; l = l->next) {
      serv->check_time = 0;     /* really check! */
      if (l->subscribed)
        _nntp_check_mailbox (NULL, l);
    }
    sidebar_draw ();
  }
  else if (!force)
    return 0;

  mutt_message _("Checking for new newsgroups...");

  now = serv->newgroups_time;
  time (&serv->newgroups_time);
  t = gmtime (&now);
  snprintf (buf, sizeof (buf), "NEWGROUPS %02d%02d%02d %02d%02d%02d GMT\r\n",
            (t->tm_year % 100), t->tm_mon + 1, t->tm_mday, t->tm_hour,
            t->tm_min, t->tm_sec);
  nntp_data.nserv = serv;
  if (Context && Context->magic == M_NNTP)
    nntp_data.group = ((nntp_data_t *) Context->data)->group;
  else
    nntp_data.group = NULL;

  emp = nntp_data_list_last(&serv->list);
  if (mutt_nntp_fetch (&nntp_data, buf, _("Adding new newsgroups..."), NULL,
                       add_group, serv, 0) != 0) {
    return -1;
  }

  mutt_message _("Loading descriptions...");

  for (l = *emp; l; l = l->next) {
    l->new = 1;
    nntp_get_desc(l, l->group, NULL, NULL);
  }
  if (*emp)
    nntp_save_cache_index(serv);
  mutt_clear_error();
  return nntp.checked;
}

/* Load list of all newsgroups from active */
int nntp_get_active (nntp_server_t * serv)
{
  char msg[STRING];
  nntp_data_t nntp_data, **tmp = &serv->list;

  if (nntp_open_connection (serv) < 0)
    return -1;

  snprintf (msg, sizeof (msg),
            _("Loading list of all newsgroups on server %s..."),
            serv->conn->account.host);
  mutt_message (msg);
  time (&serv->newgroups_time);
  nntp_data.nserv = serv;
  nntp_data.group = NULL;

  if (mutt_nntp_fetch (&nntp_data, "LIST\r\n", msg, NULL, add_group, serv, 0) < 0) {
    return -1;
  }

  m_strcpy(msg, sizeof(msg), _("Loading descriptions..."));
  mutt_message (msg);
  nntp_get_desc (&nntp_data, "*", msg, NULL);

  while (*tmp) {
    if ((*tmp)->deleted && !(*tmp)->rc) {
      nntp_data_t *d = nntp_data_list_pop(tmp);
      nntp_delete_cache(d);
      hash_remove(serv->newsgroups, d->group, NULL, NULL);
      nntp_data_delete(&d);
    } else {
      tmp = &(*tmp)->next;
    }
  }
  serv->tail = tmp;
  nntp_save_cache_index(serv);

  mutt_clear_error ();
  return nntp.checked;
}

/*
 * returns -1 if error ocurred while retrieving header,
 * number of articles which ones exist in context on success.
 */
int nntp_check_msgid (CONTEXT * ctx, const char *msgid)
{
  int ret;

  /* if msgid is already in context, don't reload them */
  if (hash_find (ctx->id_hash, msgid))
    return 1;
  if (ctx->msgcount == ctx->hdrmax)
    mx_alloc_memory (ctx);
  ctx->hdrs[ctx->msgcount] = header_new();
  ctx->hdrs[ctx->msgcount]->index = ctx->msgcount;

  mutt_message (_("Fetching %s from server..."), msgid);
  ret = nntp_read_header (ctx, msgid, 0);
  /* since nntp_read_header() may set read flag, we must reset it */
  ctx->hdrs[ctx->msgcount]->read = 0;
  if (ret != 0)
    header_delete(&ctx->hdrs[ctx->msgcount]);
  else {
    ctx->msgcount++;
    mx_update_context (ctx, 1);
    ctx->changed = 1;
  }
  return ret;
}

typedef struct {
  CONTEXT *ctx;
  int num;
  int max;
  int *child;
} CHILD_CONTEXT;

static int check_children (char *s, void *c)
{
  CHILD_CONTEXT *cc = c;
  int i, n;

  if (!s || (n = atoi (s)) == 0)
    return 0;
  for (i = 0; i < cc->ctx->msgcount; i++)
    if (cc->ctx->hdrs[i]->article_num == n)
      return 0;
  if (cc->num >= cc->max)
    p_realloc(&cc->child, cc->max += 25);
  cc->child[cc->num++] = n;

  return 0;
}

int nntp_check_children (CONTEXT * ctx, const char *msgid)
{
  nntp_data_t *nntp_data = (nntp_data_t *) ctx->data;
  char buf[STRING];
  int i, ret = 0, tmp = 0;
  CHILD_CONTEXT cc = { ctx, 0, 0, NULL };

  if (!nntp_data || !nntp_data->nserv || !nntp_data->nserv->conn ||
      !nntp_data->nserv->conn->account.host)
    return -1;
  if (nntp_data->first > nntp_data->lastLoaded)
    return 0;
  if (!nntp_data->nserv->hasXPAT) {
    mutt_error (_("Server %s does not support this operation!"),
                nntp_data->nserv->conn->account.host);
    return -1;
  }

  snprintf (buf, sizeof (buf), "XPAT References %d-%d *%s*\r\n",
            nntp_data->first, nntp_data->lastLoaded, msgid);

  if (mutt_nntp_fetch (nntp_data, buf, NULL, NULL, check_children, &cc, 0)) {
    p_delete(&cc.child);
    return -1;
  }

  /* dont try to read the xover cache. check_children() already
   * made sure that we dont have the article, so we need to visit
   * the server. Reading the cache at this point is also bad
   * because it would duplicate messages */
  if (nntp.use_cache) {
    tmp++;
    nntp.use_cache = false;
  }
  for (i = 0; i < cc.num; i++) {
    if ((ret = nntp_fetch_headers (ctx, cc.child[i], cc.child[i])))
      break;
    if (ctx->msgcount &&
        ctx->hdrs[ctx->msgcount - 1]->article_num == cc.child[i])
      ctx->hdrs[ctx->msgcount - 1]->read = 0;
  }
  if (tmp)
    nntp.use_cache = true;
  p_delete(&cc.child);
  return ret;
}

static int nntp_is_magic (const char* path, struct stat* st) {
  url_scheme_t s = url_check_scheme(NONULL(path));
  return s == U_NNTP || s == U_NNTPS ? M_NNTP : -1;
}

static int acl_check_nntp (CONTEXT* ctx, int bit) {
  switch (bit) {
    case ACL_INSERT:    /* editing messages */
    case ACL_WRITE:     /* change importance */
      return (0);
    case ACL_DELETE:    /* (un)deletion */
    case ACL_SEEN:      /* mark as read */
      return (1);
    default:
      return (0);
  }
}

mx_t const nntp_mx = {
    M_NNTP,
    0,
    nntp_is_magic,
    NULL,
    NULL,
    nntp_open_mailbox,
    NULL,
    acl_check_nntp,
    nntp_check_mailbox,
    nntp_fastclose_mailbox,
    nntp_sync_mailbox,
    NULL,
};

/* }}} */